* various major fixes

2024-03-01 22:17:44 +01:00 · 2024-03-01 22:17:44 +01:00 · f2249a624d
parent ace17a681b
commit f2249a624d
2 changed files with 93 additions and 41 deletions
--- a/README.md
+++ b/README.md
@ -30,4 +30,16 @@ source migration-env/bin/activate
 python3 -m pip install -r requirements.txt
 ```

-Then start the migration script `python3 migrate.py`.
+Then start the migration script `python3 migrate.py`.
+
+## 2024 addendum
+
+things that i fixed:
+- events are posted as the correct user
+- issue comments
+- weird unicode problems with usernames
+- option to disable the buggy joining algo
+- users are properly made admins now
+
+things that don't work but i don't care (incomplete list xD):
+- downloading media from issues
--- a/migrate.py
+++ b/migrate.py
@ -19,15 +19,21 @@ GLOBAL_ERROR_COUNT = 0
 #######################
 # CONFIG SECTION START
 #######################
-GITLAB_URL = 'https://gitlab.source.com'
-GITLAB_TOKEN = 'gitlab token'
+GITLAB_URL = 'https://...'
+GITLAB_TOKEN = ''

 # needed to clone the repositories, keep empty to try publickey (untested)
-GITLAB_ADMIN_USER = 'admin username'
-GITLAB_ADMIN_PASS = 'admin password'
+GITLAB_ADMIN_USER = ''
+GITLAB_ADMIN_PASS = ""

-GITEA_URL = 'https://gitea.dest.com'
-GITEA_TOKEN = 'gitea token'
+GITEA_URL = 'http://...'
+GITEA_TOKEN = '...'
+
+# change this to false to turn on title-based duplicate matching
+GITEA_ASSUME_CLEAN = True
+
+# if you want to force everyone to be logged through SSO, change this
+AUTH_SOURCE = 0
 #######################
 # CONFIG SECTION END
 #######################
@ -140,7 +146,7 @@ def get_user_or_group(gitea_api: pygitea, project: gitlab.v4.objects.Project) ->
    if response.ok:
        result = response.json()
    else:
-        response: requests.Response = gitea_api.get("/orgs/" + name_clean(project.namespace["name"]))
+        response: requests.Response = gitea_api.get("/orgs/" + project.namespace["path"])
        if response.ok:
            result = response.json()
        else:
@ -163,23 +169,23 @@ def get_user_keys(gitea_api: pygitea, username: string) -> {}:
 def user_exists(gitea_api: pygitea, username: string) -> bool:
    user_response: requests.Response = gitea_api.get("/users/" + username)
    if user_response.ok:
-        print_warning("User " + username + " does already exist in Gitea, skipping!")
+        print_warning("User " + username + " does already exists in Gitea, skipping!")
    else:
        print("User " + username + " not found in Gitea, importing!")

    return user_response.ok


-def user_key_exists(gitea_api: pygitea, username: string, keyname: string) -> bool:
+def user_key_exists(gitea_api: pygitea, username: string, key: string) -> bool:
    existing_keys = get_user_keys(gitea_api, username)
    if existing_keys:
-        existing_key = next((item for item in existing_keys if item["title"] == keyname), None)
+        existing_key = next((item for item in existing_keys if item["key"] == key), None)

        if existing_key is not None:
-            print_warning("Public key " + keyname + " already exists for user " + username + ", skipping!")
+            print_warning("Public key " + key + " already exists for user " + username + ", skipping!")
            return True
        else:
-            print("Public key " + keyname + " does not exists for user " + username + ", importing!")
+            print("Public key " + key + " does not exists for user " + username + ", importing!")
            return False
    else:
        print("No public keys for user " + username + ", importing!")
@ -189,7 +195,7 @@ def user_key_exists(gitea_api: pygitea, username: string, keyname: string) -> bo
 def organization_exists(gitea_api: pygitea, orgname: string) -> bool:
        group_response: requests.Response = gitea_api.get("/orgs/" + orgname)
        if group_response.ok:
-            print_warning("Group " + orgname + " does already exist in Gitea, skipping!")
+            print_warning("Group " + orgname + " does already exists in Gitea, skipping!")
        else:
            print("Group " + orgname + " not found in Gitea, importing!")

@ -215,7 +221,7 @@ def member_exists(gitea_api: pygitea, username: string, teamid: int) -> bool:
 def collaborator_exists(gitea_api: pygitea, owner: string, repo: string, username: string) -> bool:
    collaborator_response: requests.Response = gitea_api.get("/repos/" + owner + "/" + repo + "/collaborators/" + username)
    if collaborator_response.ok:
-        print_warning("Collaborator " + username + " does already exist in Gitea, skipping!")
+        print_warning("Collaborator " + username + " does already exists in Gitea, skipping!")
    else:
        print("Collaborator " + username + " not found in Gitea, importing!")

@ -225,7 +231,7 @@ def collaborator_exists(gitea_api: pygitea, owner: string, repo: string, usernam
 def repo_exists(gitea_api: pygitea, owner: string, repo: string) -> bool:
    repo_response: requests.Response = gitea_api.get("/repos/" + owner + "/" + repo)
    if repo_response.ok:
-        print_warning("Project " + repo + " does already exist in Gitea, skipping!")
+        print_warning("Project " + repo + " does already exists in Gitea, skipping!")
    else:
        print("Project " + repo + " not found in Gitea, importing!")

@ -235,13 +241,16 @@ def repo_exists(gitea_api: pygitea, owner: string, repo: string) -> bool:
 def label_exists(gitea_api: pygitea, owner: string, repo: string, labelname: string) -> bool:
    existing_labels = get_labels(gitea_api, owner, repo)
    if existing_labels:
-        existing_label = next((item for item in existing_labels if item["name"] == labelname), None)
+        if not GITEA_ASSUME_CLEAN:
+            existing_label = next((item for item in existing_labels if item["name"] == labelname), None)
+        else:
+            existing_label = None

        if existing_label is not None:
            print_warning("Label " + labelname + " already exists in project " + repo + ", skipping!")
            return True
        else:
-            print("Label " + labelname + " does not exists in project " + repo + ", importing!")
+            print("Label " + labelname + " does not exist in project " + repo + ", importing!")
            return False
    else:
        print("No labels in project " + repo + ", importing!")
@ -251,13 +260,16 @@ def label_exists(gitea_api: pygitea, owner: string, repo: string, labelname: str
 def milestone_exists(gitea_api: pygitea, owner: string, repo: string, milestone: string) -> bool:
    existing_milestones = get_milestones(gitea_api, owner, repo)
    if existing_milestones:
-        existing_milestone = next((item for item in existing_milestones if item["title"] == milestone), None)
+        if not GITEA_ASSUME_CLEAN:
+            existing_milestone = next((item for item in existing_milestones if item["title"] == milestone), None)
+        else:
+            existing_milestone = None

        if existing_milestone is not None:
            print_warning("Milestone " + milestone + " already exists in project " + repo + ", skipping!")
            return True
        else:
-            print("Milestone " + milestone + " does not exists in project " + repo + ", importing!")
+            print("Milestone " + milestone + " does not exist in project " + repo + ", importing!")
            return False
    else:
        print("No milestones in project " + repo + ", importing!")
@ -267,13 +279,16 @@ def milestone_exists(gitea_api: pygitea, owner: string, repo: string, milestone:
 def issue_exists(gitea_api: pygitea, owner: string, repo: string, issue: string) -> bool:
    existing_issues = get_issues(gitea_api, owner, repo)
    if existing_issues:
-        existing_issue = next((item for item in existing_issues if item["title"] == issue), None)
+        if not GITEA_ASSUME_CLEAN:
+            existing_issue = next((item for item in existing_issues if item["title"] == issue), None)
+        else:
+            existing_issue = None

        if existing_issue is not None:
            print_warning("Issue " + issue + " already exists in project " + repo + ", skipping!")
            return True
        else:
-            print("Issue " + issue + " does not exists in project " + repo + ", importing!")
+            print("Issue " + issue + " does not exist in project " + repo + ", importing!")
            return False
    else:
        print("No issues in project " + repo + ", importing!")
@ -371,15 +386,31 @@ def _import_project_issues(gitea_api: pygitea, issues: [gitlab.v4.objects.Projec
                "labels": labels,
                "milestone": milestone,
                "title": issue.title,
+            }, params={
+                "sudo": issue.author["username"],
            })
            if import_response.ok:
                print_info("Issue " + issue.title + " imported!")
+                if len(issue.notes.list()) > 0:
+                    index = import_response.json()["number"]
+                    for note in issue.notes.list():
+                        import_response: requests.Response = gitea_api.post("/repos/" + owner + "/" + repo + "/issues/" + str(index) + "/comments" , json={
+                            "body": note.body,
+                        }, params={
+                            "sudo": note.author["username"],
+                        })
+                        if import_response.ok:
+                            print_info("reply to " + issue.title + " imported!")
+                        else:
+                            print_error("reply to " + issue.title + " import failed: " + import_response.text)
+
            else:
                print_error("Issue " + issue.title + " import failed: " + import_response.text)
+        


 def _import_project_repo(gitea_api: pygitea, project: gitlab.v4.objects.Project):
-    if not repo_exists(gitea_api, project.namespace['name'], name_clean(project.name)):
+    if not repo_exists(gitea_api, project.namespace['path'], project.path):
        clone_url = project.http_url_to_repo
        if GITLAB_ADMIN_PASS == '' and GITLAB_ADMIN_USER == '':
            clone_url = project.ssh_url_to_repo
@ -395,21 +426,21 @@ def _import_project_repo(gitea_api: pygitea, project: gitlab.v4.objects.Project)
                "description": project.description,
                "mirror": False,
                "private": private,
-                "repo_name": name_clean(project.name),
+                "repo_name": project.path, # <-- for URL compat; use `name_clean(project.name)` for pretty URLs
                "uid": owner['id']
            })
            if import_response.ok:
-                print_info("Project " + name_clean(project.name) + " imported!")
+                print_info("Project " + name_clean(project.path) + " imported!")
            else:
-                print_error("Project " + name_clean(project.name) + " import failed: " + import_response.text)
+                print_error("Project " + name_clean(project.path) + " import failed: " + import_response.text)
        else:
-            print_error("Failed to load project owner for project " + name_clean(project.name))
+            print_error("Failed to load project owner for project " + project.path)


 def _import_project_repo_collaborators(gitea_api: pygitea, collaborators: [gitlab.v4.objects.ProjectMember], project: gitlab.v4.objects.Project):
    for collaborator in collaborators:
        
-        if not collaborator_exists(gitea_api, project.namespace['name'], name_clean(project.name), collaborator.username):
+        if not collaborator_exists(gitea_api, project.namespace['path'], project.path, collaborator.username):
            permission = "read"
            
            if collaborator.access_level == 10:    # guest access
@ -421,12 +452,12 @@ def _import_project_repo_collaborators(gitea_api: pygitea, collaborators: [gitla
            elif collaborator.access_level == 40:  # maintainer access
                permission = "admin"
            elif collaborator.access_level == 50:  # owner access (only for groups)
-                print_error("Groupmembers are currently not supported!")
+                print_warning("Groupmembers are currently not supported!")
                continue  # groups are not supported
            else:
                print_warning("Unsupported access level " + str(collaborator.access_level) + ", setting permissions to 'read'!")
            
-            import_response: requests.Response = gitea_api.put("/repos/" + project.namespace['name'] +"/" + name_clean(project.name) + "/collaborators/" + collaborator.username, json={
+            import_response: requests.Response = gitea_api.put("/repos/" + project.namespace['path'] +"/" + project.path + "/collaborators/" + collaborator.username, json={
                "permission": permission
            })
            if import_response.ok:
@ -443,7 +474,7 @@ def _import_users(gitea_api: pygitea, users: [gitlab.v4.objects.User], notify: b
        print("Found " + str(len(keys)) + " public keys for user " + user.username)

        if not user_exists(gitea_api, user.username):
-            tmp_password = 'Tmp1!' + ''.join(random.choices(string.ascii_uppercase + string.digits, k=10))
+            tmp_password = 'Tmp1!' + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=40))
            tmp_email = user.username + '@noemail-git.local'  # Some gitlab instances do not publish user emails
            try:
                tmp_email = user.email
@ -455,11 +486,20 @@ def _import_users(gitea_api: pygitea, users: [gitlab.v4.objects.User], notify: b
                "login_name": user.username,
                "password": tmp_password,
                "send_notify": notify,
-                "source_id": 0, # local user
+                "source_id": AUTH_SOURCE, # 0 = local user
                "username": user.username
            })
            if import_response.ok:
                print_info("User " + user.username + " imported, temporary password: " + tmp_password)
+                if user.is_admin:
+                    import_response: requests.Response = gitea_api.patch("/admin/users/" + user.username, json={
+                    	"admin": True,
+                    	"login_name": user.username
+                    })
+                    if import_response.ok:
+                        print_info("made" + user.username + " an admin")
+                    else:
+                        print_error("failed making " + user.username + " an admin: " + import_response.text)
            else:
                print_error("User " + user.username + " import failed: " + import_response.text)
        
@ -469,7 +509,7 @@ def _import_users(gitea_api: pygitea, users: [gitlab.v4.objects.User], notify: b

 def _import_user_keys(gitea_api: pygitea, keys: [gitlab.v4.objects.UserKey], user: gitlab.v4.objects.User):
    for key in keys:
-        if not user_key_exists(gitea_api, user.username, key.title):
+        if not user_key_exists(gitea_api, user.username, key.key):
            import_response: requests.Response = gitea_api.post("/admin/users/" + user.username + "/keys", json={
                "key": key.key,
                "read_only": True,
@ -493,13 +533,13 @@ def _import_groups(gitea_api: pygitea, groups: [gitlab.v4.objects.Group]):
                "description": group.description,
                "full_name": group.full_name,
                "location": "",
-                "username": name_clean(group.name),
+                "username": group.path,
                "website": ""
            })
            if import_response.ok:
-                print_info("Group " + name_clean(group.name) + " imported!")
+                print_info("Group " + group.path + " imported!")
            else:
-                print_error("Group " + name_clean(group.name) + " import failed: " + import_response.text)
+                print_error("Group " + group.name + " import failed: " + import_response.text)

        # import group members
        _import_group_members(gitea_api, members, group)
@ -507,7 +547,7 @@ def _import_groups(gitea_api: pygitea, groups: [gitlab.v4.objects.Group]):

 def _import_group_members(gitea_api: pygitea, members: [gitlab.v4.objects.GroupMember], group: gitlab.v4.objects.Group):
    # TODO: create teams based on gitlab permissions (access_level of group member)
-    existing_teams = get_teams(gitea_api, name_clean(group.name))
+    existing_teams = get_teams(gitea_api, group.path)
    if existing_teams:
        first_team = existing_teams[0]
        print("Organization teams fetched, importing users to first team: " + first_team['name'])
@ -530,7 +570,7 @@ def _import_group_members(gitea_api: pygitea, members: [gitlab.v4.objects.GroupM

 def import_users_groups(gitlab_api: gitlab.Gitlab, gitea_api: pygitea, notify=False):
    # read all users
-    users: [gitlab.v4.objects.User] = gitlab_api.users.list(all=True)
+    users: [gitlab.v4.objects.User] = gitlab_api.users.list(active=True, all=True)
    groups: [gitlab.v4.objects.Group] = gitlab_api.groups.list(all=True)

    print("Found " + str(len(users)) + " gitlab users as user " + gitlab_api.user.username)
@ -568,13 +608,13 @@ def import_projects(gitlab_api: gitlab.Gitlab, gitea_api: pygitea):
        _import_project_repo_collaborators(gitea_api, collaborators, project)

        # import labels
-        _import_project_labels(gitea_api, labels, project.namespace['name'], name_clean(project.name))
+        _import_project_labels(gitea_api, labels, project.namespace['path'], project.path)

        # import milestones
-        _import_project_milestones(gitea_api, milestones, project.namespace['name'], name_clean(project.name))
+        _import_project_milestones(gitea_api, milestones, project.namespace['path'], project.path)

        # import issues
-        _import_project_issues(gitea_api, issues, project.namespace['name'], name_clean(project.name))
+        _import_project_issues(gitea_api, issues, project.namespace['path'], project.path)


 #