* various major fixes

Add a "tested versions" part

README.md

@@ -12,8 +12,34 @@ This script support migrating the following data:
  - Groups
  - Public SSH keys
 
+Tested with Gitlab Version 13.0.6 and Gitea Version 1.11.6.
+
 ## Usage
 Change items in the config section of the script.
 
 Install all dependencies via `python -m pip install -r requirements.txt` and
 use python3 to execute the script.
+
+### How to use with venv
+To keep your local system clean, it might be helpful to store all Python dependencies in one folder.
+Python provides a virtual environment package which can be used to accomplish this task.
+
+```bash
+python3 -m venv migration-env
+source migration-env/bin/activate
+python3 -m pip install -r requirements.txt
+```
+
+Then start the migration script `python3 migrate.py`.
+
+## 2024 addendum
+
+things that i fixed:
+- events are posted as the correct user
+- issue comments
+- weird unicode problems with usernames
+- option to disable the buggy joining algo
+- users are properly made admins now
+
+things that don't work but i don't care (incomplete list xD):
+- downloading media from issues

migrate.py

@@ -7,6 +7,7 @@ import requests
 import json
 import dateutil.parser
 import datetime
+import re
 
 import gitlab  # pip install python-gitlab
 import gitlab.v4.objects
@@ -18,15 +19,21 @@ GLOBAL_ERROR_COUNT = 0
 #######################
 # CONFIG SECTION START
 #######################
-GITLAB_URL = 'https://gitlab.source.com'
-GITLAB_TOKEN = 'gitlab token'
+GITLAB_URL = 'https://...'
+GITLAB_TOKEN = ''
 
 # needed to clone the repositories, keep empty to try publickey (untested)
-GITLAB_ADMIN_USER = 'admin username'
-GITLAB_ADMIN_PASS = 'admin password'
+GITLAB_ADMIN_USER = ''
+GITLAB_ADMIN_PASS = ""
 
-GITEA_URL = 'https://gitea.dest.com'
-GITEA_TOKEN = 'gitea token'
+GITEA_URL = 'http://...'
+GITEA_TOKEN = '...'
+
+# change this to false to turn on title-based duplicate matching
+GITEA_ASSUME_CLEAN = True
+
+# if you want to force everyone to be logged through SSO, change this
+AUTH_SOURCE = 0
 #######################
 # CONFIG SECTION END
 #######################
@@ -41,7 +48,7 @@ def main():
     gl = gitlab.Gitlab(GITLAB_URL, private_token=GITLAB_TOKEN)
     gl.auth()
     assert(isinstance(gl.user, gitlab.v4.objects.CurrentUser))
-
+    print_info("Connected to Gitlab, version: " + str(gl.version()))
 
     gt = pygitea.API(GITEA_URL, token=GITEA_TOKEN)
     gt_version = gt.get('/version').json()
@@ -133,13 +140,17 @@ def get_collaborators(gitea_api: pygitea, owner: string, repo: string) -> []:
     return existing_collaborators
 
 
-def get_user_or_group(gitea_api: pygitea, name: string) -> {}:
+def get_user_or_group(gitea_api: pygitea, project: gitlab.v4.objects.Project) -> {}:
     result = None
-    response: requests.Response = gitea_api.get("/users/" + name)
+    response: requests.Response = gitea_api.get("/users/" + project.namespace['path'])
     if response.ok:
         result = response.json()
     else:
-        print_error("Failed to load user or group " + name + "! " + response.text)
+        response: requests.Response = gitea_api.get("/orgs/" + project.namespace["path"])
+        if response.ok:
+            result = response.json()
+        else:
+            print_error("Failed to load user or group " + project.namespace["name"] + "! " + response.text)
 
     return result
 
@@ -158,23 +169,23 @@ def get_user_keys(gitea_api: pygitea, username: string) -> {}:
 def user_exists(gitea_api: pygitea, username: string) -> bool:
     user_response: requests.Response = gitea_api.get("/users/" + username)
     if user_response.ok:
-        print_warning("User " + username + " does already exist in Gitea, skipping!")
+        print_warning("User " + username + " does already exists in Gitea, skipping!")
     else:
         print("User " + username + " not found in Gitea, importing!")
 
     return user_response.ok
 
 
-def user_key_exists(gitea_api: pygitea, username: string, keyname: string) -> bool:
+def user_key_exists(gitea_api: pygitea, username: string, key: string) -> bool:
     existing_keys = get_user_keys(gitea_api, username)
     if existing_keys:
-        existing_key = next((item for item in existing_keys if item["title"] == keyname), None)
+        existing_key = next((item for item in existing_keys if item["key"] == key), None)
 
         if existing_key is not None:
-            print_warning("Public key " + keyname + " already exists for user " + username + ", skipping!")
+            print_warning("Public key " + key + " already exists for user " + username + ", skipping!")
             return True
         else:
-            print("Public key " + keyname + " does not exists for user " + username + ", importing!")
+            print("Public key " + key + " does not exists for user " + username + ", importing!")
             return False
     else:
         print("No public keys for user " + username + ", importing!")
@@ -184,7 +195,7 @@ def user_key_exists(gitea_api: pygitea, username: string, keyname: string) -> bo
 def organization_exists(gitea_api: pygitea, orgname: string) -> bool:
         group_response: requests.Response = gitea_api.get("/orgs/" + orgname)
         if group_response.ok:
-            print_warning("Group " + orgname + " does already exist in Gitea, skipping!")
+            print_warning("Group " + orgname + " does already exists in Gitea, skipping!")
         else:
             print("Group " + orgname + " not found in Gitea, importing!")
 
@@ -210,7 +221,7 @@ def member_exists(gitea_api: pygitea, username: string, teamid: int) -> bool:
 def collaborator_exists(gitea_api: pygitea, owner: string, repo: string, username: string) -> bool:
     collaborator_response: requests.Response = gitea_api.get("/repos/" + owner + "/" + repo + "/collaborators/" + username)
     if collaborator_response.ok:
-        print_warning("Collaborator " + username + " does already exist in Gitea, skipping!")
+        print_warning("Collaborator " + username + " does already exists in Gitea, skipping!")
     else:
         print("Collaborator " + username + " not found in Gitea, importing!")
 
@@ -220,7 +231,7 @@ def collaborator_exists(gitea_api: pygitea, owner: string, repo: string, usernam
 def repo_exists(gitea_api: pygitea, owner: string, repo: string) -> bool:
     repo_response: requests.Response = gitea_api.get("/repos/" + owner + "/" + repo)
     if repo_response.ok:
-        print_warning("Project " + repo + " does already exist in Gitea, skipping!")
+        print_warning("Project " + repo + " does already exists in Gitea, skipping!")
     else:
         print("Project " + repo + " not found in Gitea, importing!")
 
@@ -230,13 +241,16 @@ def repo_exists(gitea_api: pygitea, owner: string, repo: string) -> bool:
 def label_exists(gitea_api: pygitea, owner: string, repo: string, labelname: string) -> bool:
     existing_labels = get_labels(gitea_api, owner, repo)
     if existing_labels:
-        existing_label = next((item for item in existing_labels if item["name"] == labelname), None)
+        if not GITEA_ASSUME_CLEAN:
+            existing_label = next((item for item in existing_labels if item["name"] == labelname), None)
+        else:
+            existing_label = None
 
         if existing_label is not None:
             print_warning("Label " + labelname + " already exists in project " + repo + ", skipping!")
             return True
         else:
-            print("Label " + labelname + " does not exists in project " + repo + ", importing!")
+            print("Label " + labelname + " does not exist in project " + repo + ", importing!")
             return False
     else:
         print("No labels in project " + repo + ", importing!")
@@ -246,13 +260,16 @@ def label_exists(gitea_api: pygitea, owner: string, repo: string, labelname: str
 def milestone_exists(gitea_api: pygitea, owner: string, repo: string, milestone: string) -> bool:
     existing_milestones = get_milestones(gitea_api, owner, repo)
     if existing_milestones:
-        existing_milestone = next((item for item in existing_milestones if item["title"] == milestone), None)
+        if not GITEA_ASSUME_CLEAN:
+            existing_milestone = next((item for item in existing_milestones if item["title"] == milestone), None)
+        else:
+            existing_milestone = None
 
         if existing_milestone is not None:
             print_warning("Milestone " + milestone + " already exists in project " + repo + ", skipping!")
             return True
         else:
-            print("Milestone " + milestone + " does not exists in project " + repo + ", importing!")
+            print("Milestone " + milestone + " does not exist in project " + repo + ", importing!")
             return False
     else:
         print("No milestones in project " + repo + ", importing!")
@@ -262,13 +279,16 @@ def milestone_exists(gitea_api: pygitea, owner: string, repo: string, milestone:
 def issue_exists(gitea_api: pygitea, owner: string, repo: string, issue: string) -> bool:
     existing_issues = get_issues(gitea_api, owner, repo)
     if existing_issues:
-        existing_issue = next((item for item in existing_issues if item["title"] == issue), None)
+        if not GITEA_ASSUME_CLEAN:
+            existing_issue = next((item for item in existing_issues if item["title"] == issue), None)
+        else:
+            existing_issue = None
 
         if existing_issue is not None:
             print_warning("Issue " + issue + " already exists in project " + repo + ", skipping!")
             return True
         else:
-            print("Issue " + issue + " does not exists in project " + repo + ", importing!")
+            print("Issue " + issue + " does not exist in project " + repo + ", importing!")
             return False
     else:
         print("No issues in project " + repo + ", importing!")
@@ -366,22 +386,38 @@ def _import_project_issues(gitea_api: pygitea, issues: [gitlab.v4.objects.Projec
                 "labels": labels,
                 "milestone": milestone,
                 "title": issue.title,
+            }, params={
+                "sudo": issue.author["username"],
             })
             if import_response.ok:
                 print_info("Issue " + issue.title + " imported!")
+                if len(issue.notes.list()) > 0:
+                    index = import_response.json()["number"]
+                    for note in issue.notes.list():
+                        import_response: requests.Response = gitea_api.post("/repos/" + owner + "/" + repo + "/issues/" + str(index) + "/comments" , json={
+                            "body": note.body,
+                        }, params={
+                            "sudo": note.author["username"],
+                        })
+                        if import_response.ok:
+                            print_info("reply to " + issue.title + " imported!")
+                        else:
+                            print_error("reply to " + issue.title + " import failed: " + import_response.text)
+
             else:
                 print_error("Issue " + issue.title + " import failed: " + import_response.text)
+        
 
 
 def _import_project_repo(gitea_api: pygitea, project: gitlab.v4.objects.Project):
-    if not repo_exists(gitea_api, project.namespace['name'], project.name):
+    if not repo_exists(gitea_api, project.namespace['path'], project.path):
         clone_url = project.http_url_to_repo
-        if GITLAB_ADMIN_PASS is '' and GITLAB_ADMIN_USER is '':
+        if GITLAB_ADMIN_PASS == '' and GITLAB_ADMIN_USER == '':
             clone_url = project.ssh_url_to_repo
         private = project.visibility == 'private' or project.visibility == 'internal'
 
         # Load the owner (users and groups can both be fetched using the /users/ endpoint)
-        owner = get_user_or_group(gitea_api, project.namespace['name'])
+        owner = get_user_or_group(gitea_api, project)
         if owner:
             import_response: requests.Response = gitea_api.post("/repos/migrate", json={
                 "auth_password": GITLAB_ADMIN_PASS,
@@ -390,21 +426,21 @@ def _import_project_repo(gitea_api: pygitea, project: gitlab.v4.objects.Project)
                 "description": project.description,
                 "mirror": False,
                 "private": private,
-                "repo_name": project.name,
+                "repo_name": project.path, # <-- for URL compat; use `name_clean(project.name)` for pretty URLs
                 "uid": owner['id']
             })
             if import_response.ok:
-                print_info("Project " + project.name + " imported!")
+                print_info("Project " + name_clean(project.path) + " imported!")
             else:
-                print_error("Project " + project.name + " import failed: " + import_response.text)
+                print_error("Project " + name_clean(project.path) + " import failed: " + import_response.text)
         else:
-            print_error("Failed to load project owner for project " + project.name)
+            print_error("Failed to load project owner for project " + project.path)
 
 
 def _import_project_repo_collaborators(gitea_api: pygitea, collaborators: [gitlab.v4.objects.ProjectMember], project: gitlab.v4.objects.Project):
     for collaborator in collaborators:
         
-        if not collaborator_exists(gitea_api, project.namespace['name'], project.name, collaborator.username):
+        if not collaborator_exists(gitea_api, project.namespace['path'], project.path, collaborator.username):
             permission = "read"
             
             if collaborator.access_level == 10:    # guest access
@@ -416,12 +452,12 @@ def _import_project_repo_collaborators(gitea_api: pygitea, collaborators: [gitla
             elif collaborator.access_level == 40:  # maintainer access
                 permission = "admin"
             elif collaborator.access_level == 50:  # owner access (only for groups)
-                print_error("Groupmembers are currently not supported!")
+                print_warning("Groupmembers are currently not supported!")
                 continue  # groups are not supported
             else:
                 print_warning("Unsupported access level " + str(collaborator.access_level) + ", setting permissions to 'read'!")
             
-            import_response: requests.Response = gitea_api.put("/repos/" + project.namespace['name'] +"/" + project.name + "/collaborators/" + collaborator.username, json={
+            import_response: requests.Response = gitea_api.put("/repos/" + project.namespace['path'] +"/" + project.path + "/collaborators/" + collaborator.username, json={
                 "permission": permission
             })
             if import_response.ok:
@@ -438,18 +474,32 @@ def _import_users(gitea_api: pygitea, users: [gitlab.v4.objects.User], notify: b
         print("Found " + str(len(keys)) + " public keys for user " + user.username)
 
         if not user_exists(gitea_api, user.username):
-            tmp_password = ''.join(random.choices(string.ascii_uppercase + string.digits, k=10))
+            tmp_password = 'Tmp1!' + ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=40))
+            tmp_email = user.username + '@noemail-git.local'  # Some gitlab instances do not publish user emails
+            try:
+                tmp_email = user.email
+            except AttributeError:
+                pass
             import_response: requests.Response = gitea_api.post("/admin/users", json={
-                "email": user.email,
+                "email": tmp_email,
                 "full_name": user.name,
                 "login_name": user.username,
                 "password": tmp_password,
                 "send_notify": notify,
-                "source_id": 0, # local user
+                "source_id": AUTH_SOURCE, # 0 = local user
                 "username": user.username
             })
             if import_response.ok:
                 print_info("User " + user.username + " imported, temporary password: " + tmp_password)
+                if user.is_admin:
+                    import_response: requests.Response = gitea_api.patch("/admin/users/" + user.username, json={
+                    	"admin": True,
+                    	"login_name": user.username
+                    })
+                    if import_response.ok:
+                        print_info("made" + user.username + " an admin")
+                    else:
+                        print_error("failed making " + user.username + " an admin: " + import_response.text)
             else:
                 print_error("User " + user.username + " import failed: " + import_response.text)
         
@@ -459,7 +509,7 @@ def _import_users(gitea_api: pygitea, users: [gitlab.v4.objects.User], notify: b
 
 def _import_user_keys(gitea_api: pygitea, keys: [gitlab.v4.objects.UserKey], user: gitlab.v4.objects.User):
     for key in keys:
-        if not user_key_exists(gitea_api, user.username, key.title):
+        if not user_key_exists(gitea_api, user.username, key.key):
             import_response: requests.Response = gitea_api.post("/admin/users/" + user.username + "/keys", json={
                 "key": key.key,
                 "read_only": True,
@@ -475,19 +525,19 @@ def _import_groups(gitea_api: pygitea, groups: [gitlab.v4.objects.Group]):
     for group in groups:
         members: [gitlab.v4.objects.GroupMember] = group.members.list(all=True)
 
-        print("Importing group " + group.name + "...")
-        print("Found " + str(len(members)) + " gitlab members for group " + group.name)
+        print("Importing group " + name_clean(group.name) + "...")
+        print("Found " + str(len(members)) + " gitlab members for group " + name_clean(group.name))
 
-        if not organization_exists(gitea_api, group.name):
+        if not organization_exists(gitea_api, name_clean(group.name)):
             import_response: requests.Response = gitea_api.post("/orgs", json={
                 "description": group.description,
                 "full_name": group.full_name,
                 "location": "",
-                "username": group.name,
+                "username": group.path,
                 "website": ""
             })
             if import_response.ok:
-                print_info("Group " + group.name + " imported!")
+                print_info("Group " + group.path + " imported!")
             else:
                 print_error("Group " + group.name + " import failed: " + import_response.text)
 
@@ -497,7 +547,7 @@ def _import_groups(gitea_api: pygitea, groups: [gitlab.v4.objects.Group]):
 
 def _import_group_members(gitea_api: pygitea, members: [gitlab.v4.objects.GroupMember], group: gitlab.v4.objects.Group):
     # TODO: create teams based on gitlab permissions (access_level of group member)
-    existing_teams = get_teams(gitea_api, group.name)
+    existing_teams = get_teams(gitea_api, group.path)
     if existing_teams:
         first_team = existing_teams[0]
         print("Organization teams fetched, importing users to first team: " + first_team['name'])
@@ -507,11 +557,11 @@ def _import_group_members(gitea_api: pygitea, members: [gitlab.v4.objects.GroupM
             if not member_exists(gitea_api, member.username, first_team['id']):
                 import_response: requests.Response = gitea_api.put("/teams/" + str(first_team['id']) + "/members/" + member.username)
                 if import_response.ok:
-                    print_info("Member " + member.username + " added to group " + group.name + "!")
+                    print_info("Member " + member.username + " added to group " + name_clean(group.name) + "!")
                 else:
-                    print_error("Failed to add member " + member.username + " to group " + group.name + "!")
+                    print_error("Failed to add member " + member.username + " to group " + name_clean(group.name) + "!")
     else:
-        print_error("Failed to import members to group " + group.name + ": no teams found!")
+        print_error("Failed to import members to group " + name_clean(group.name) + ": no teams found!")
 
 
 #
@@ -520,7 +570,7 @@ def _import_group_members(gitea_api: pygitea, members: [gitlab.v4.objects.GroupM
 
 def import_users_groups(gitlab_api: gitlab.Gitlab, gitea_api: pygitea, notify=False):
     # read all users
-    users: [gitlab.v4.objects.User] = gitlab_api.users.list(all=True)
+    users: [gitlab.v4.objects.User] = gitlab_api.users.list(active=True, all=True)
     groups: [gitlab.v4.objects.Group] = gitlab_api.groups.list(all=True)
 
     print("Found " + str(len(users)) + " gitlab users as user " + gitlab_api.user.username)
@@ -545,11 +595,11 @@ def import_projects(gitlab_api: gitlab.Gitlab, gitea_api: pygitea):
         milestones: [gitlab.v4.objects.ProjectMilestone] = project.milestones.list(all=True)
         issues: [gitlab.v4.objects.ProjectIssue] = project.issues.list(all=True)
 
-        print("Importing project " + project.name + " from owner " + project.namespace['name'])
-        print("Found " + str(len(collaborators)) + " collaborators for project " + project.name)
-        print("Found " + str(len(labels)) + " labels for project " + project.name)
-        print("Found " + str(len(milestones)) + " milestones for project " + project.name)
-        print("Found " + str(len(issues)) + " issues for project " + project.name)
+        print("Importing project " + name_clean(project.name) + " from owner " + project.namespace['name'])
+        print("Found " + str(len(collaborators)) + " collaborators for project " + name_clean(project.name))
+        print("Found " + str(len(labels)) + " labels for project " + name_clean(project.name))
+        print("Found " + str(len(milestones)) + " milestones for project " + name_clean(project.name))
+        print("Found " + str(len(issues)) + " issues for project " + name_clean(project.name))
 
         # import project repo
         _import_project_repo(gitea_api, project)
@@ -558,13 +608,13 @@ def import_projects(gitlab_api: gitlab.Gitlab, gitea_api: pygitea):
         _import_project_repo_collaborators(gitea_api, collaborators, project)
 
         # import labels
-        _import_project_labels(gitea_api, labels, project.namespace['name'], project.name)
+        _import_project_labels(gitea_api, labels, project.namespace['path'], project.path)
 
         # import milestones
-        _import_project_milestones(gitea_api, milestones, project.namespace['name'], project.name)
+        _import_project_milestones(gitea_api, milestones, project.namespace['path'], project.path)
 
         # import issues
-        _import_project_issues(gitea_api, issues, project.namespace['name'], project.name)
+        _import_project_issues(gitea_api, issues, project.namespace['path'], project.path)
 
 
 #
@@ -610,5 +660,15 @@ def print_error(message):
     print_color(bcolors.FAIL, message)
 
 
+def name_clean(name):
+    newName = name.replace(" ", "_")
+    newName = re.sub(r"[^a-zA-Z0-9_\.-]", "-", newName)
+
+    if (newName.lower() == "plugins"):
+        return newName + "-user"
+    
+    return newName
+
+
 if __name__ == "__main__":
     main()