weblate/ci/run-migrate

#!/bin/sh

# Copyright © Michal Čihař <michal@weblate.org>
#
# SPDX-License-Identifier: GPL-3.0-or-later

# Migrations test executor

. ci/lib.sh

if [ -n "$1" ]; then
    TAG="weblate-$1"
else
    echo "Missing version to migrate from!"
    exit 1
fi

HEAD_COMMIT=$(git rev-parse HEAD)

# Copy the current file to survive checkout
cp ci/migrate-override.txt /tmp/migrate-override.txt

print_step "Testing migration from $TAG on $CI_DATABASE..."
cleanup_database
check
if ! git fetch --depth 1 origin tag "$TAG"; then
    git remote add upstream https://github.com/WeblateOrg/weblate.git
    git fetch --depth 1 upstream tag "$TAG"
fi
check
git checkout "$TAG"
check
# Use clean virtualenv for each version, this avoids problems when trying to downgrade from current versions
uv venv --python python3.11 ".venv-$TAG"
check
# shellcheck source=/dev/null
. ".venv-$TAG/bin/activate"
# Install matching Weblate package
# - Need to keep "ci" as Weblate 5.8 and older have the dependency separately
# - Force PyGobject 3.52 to be compatible with girepository-2.0
uv pip install --override /tmp/migrate-override.txt -e ".[all,ci,mysql]"
check
echo "DATABASES['default']['HOST'] = '$CI_DB_HOST'" >> weblate/settings_test.py
check
if [ -n "$CI_DB_PASSWORD" ]; then
    echo "DATABASES['default']['PASSWORD'] = '$CI_DB_PASSWORD'" >> weblate/settings_test.py
    check
fi
if [ -n "$CI_DB_PORT" ]; then
    echo "DATABASES['default']['PORT'] = '$CI_DB_PORT'" >> weblate/settings_test.py
    check
fi
./manage.py migrate
check

# Delete automatically created languages to be able to load fixture
./manage.py shell -c 'from weblate.lang.models import Language; Language.objects.all().delete()'
check

# Load basic project fixture from the older version
./manage.py loaddata simple-project.json
check

# Force creating project groups
./manage.py shell -c 'from weblate.trans.models import Project; [project.save() for project in Project.objects.iterator()]'
check
# Enable suggestion voting
./manage.py shell -c 'from weblate.trans.models import Component; Component.objects.all().update(suggestion_voting=True, suggestion_autoaccept=2)'
check
# Add suggestions
./manage.py add_suggestions test test cs weblate/trans/tests/data/cs.po
check
# Add vote for suggestion
./manage.py shell -c 'from weblate.trans.models import Vote, Suggestion; s = Suggestion.objects.all()[0]; vote = Vote(suggestion=s, user=s.user); vote.value = 1; vote.positive = True; vote.save()'
check
# Add a global metric
./manage.py shell -c 'from weblate.metrics.models import METRIC_ORDER, Metric; Metric.objects.create(scope=Metric.SCOPE_GLOBAL, data=[1] * len(METRIC_ORDER), relation=0, changes=1)'
check
# Load translation memory
./manage.py import_memory ./weblate/trans/tests/data/memory.json
check
# Create huge translation memory entry
./manage.py shell -c 'from weblate.memory.models import Memory; from weblate.lang.models import Language; Memory.objects.create(source_language=Language.objects.get(code="en"), target_language=Language.objects.get(code="cs"), source="source"*1000, target="target"*1000, origin="origin"*1000)'
check

# Add a pending unit
semver_compare "$1" "<" "5.12"
pending_migration_test=$?
if [ "$pending_migration_test" -eq 0 ]; then
    ./manage.py shell << EOF
from django.contrib.auth import get_user_model
from weblate.trans.models import Unit, Change
from weblate.utils.state import STATE_TRANSLATED
User = get_user_model()
user = User.objects.create(username="migratetest")
unit = Unit.objects.all()[0]
unit.target = "Test Target"
unit.explanation = "Test Explanation"
unit.state = STATE_TRANSLATED
unit.save()
Change.objects.create(translation=unit.translation, user=user, action=1, unit=unit, target=unit.target)
unit.pending = True
unit.save()
EOF
    check
fi

semver_compare "$1" ">" "5.5"
addon_scopes=$?
# Create addons with different scopes
semver_compare "$1" "<" "5.12"
file_format_params=$?

if [ "$addon_scopes" -eq 0 ] && [ "$file_format_params" -eq 0 ]; then
    ./manage.py shell << EOF
from weblate.addons.models import Addon
from weblate.trans.models import Component, Project
Addon.objects.bulk_create([
	Addon(name="weblate.gettext.msgmerge", configuration={"previous": False}),
	Addon(project_id=1, name="weblate.gettext.customize", configuration={"width": -1}),
	Addon(component_id=1, name="weblate.gettext.msgmerge", configuration={"no_location": True}),
	Addon(name="weblate.yaml.customize", configuration={"line_break": "dos"}),
	Addon(project_id=1, name="weblate.yaml.customize", configuration={"indent": 2}),
	Addon(component_id=1, name="weblate.json.customize", configuration={"style": "spaces"}),
])
EOF
    check
fi

git reset --hard
check
git checkout "$HEAD_COMMIT"
check
# Use CI environment
deactivate
check
run_coverage ./manage.py migrate
check
# Check migrated vote exists
uv run --all-extras ./manage.py shell -c 'from weblate.trans.models import Vote; Vote.objects.get(value=1)'
check

# Check migrated pending unit
if [ "$pending_migration_test" -eq 0 ]; then
    uv run --all-extras ./manage.py shell << EOF
from weblate.trans.models import PendingUnitChange

change = PendingUnitChange.objects.get()
unit = change.unit

errors = []
if change.target != unit.target:
    errors.append(f"Target mismatch: '{change.target}' != '{unit.target}'")
if change.explanation != unit.explanation:
    errors.append(f"Explanation mismatch: '{change.explanation}' != '{unit.explanation}'")
if change.state != unit.state:
    errors.append(f"State mismatch: {change.state} != {unit.state}")
# The migration uses get_last_content_change to get author
author = unit.get_last_content_change()[0]
if change.author != author:
    errors.append(f"Author mismatch: {change.author} != {author}")
if change.source_unit_explanation != unit.source_unit.explanation:
    errors.append(f"Source unit explanation mismatch: '{change.source_unit_explanation}' != '{unit.source_unit.explanation}'")
assert not errors, "\n".join(errors)
EOF
    check
fi

if [ "$addon_scopes" -eq 0 ] && [ "$file_format_params" -eq 0 ]; then
    # check that relevant addon parameters have been migrated to file_format_params
    uv run --all-extras ./manage.py shell << EOF
from weblate.addons.models import Addon
from weblate.trans.models import Component
file_format_params = Component.objects.get(pk=1).file_format_params
# check site wide params are migrated
assert 'yaml_line_break' not in file_format_params
assert 'yaml_indent' not in file_format_params
assert 'json_indent_style' not in file_format_params
assert file_format_params['po_line_wrap'] == -1
assert file_format_params['po_keep_previous'] is False
assert file_format_params['po_no_location'] is True
assert not Addon.objects.filter(name="weblate.json.customize").exists()
assert not Addon.objects.filter(name="weblate.gettext.customize").exists()
assert Addon.objects.filter(name="weblate.gettext.msgmerge").exists()
EOF
    check
fi