refs #2371 hide commits that can't be restored

api/repo_api.py

@@ -56,11 +56,10 @@ config_file = '/opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg'
 
 REPOSITORIES_BASE_PATH = "/opt/opengnsys/ogrepository/oggit/git/"
 OGGIT_USER = "oggit"
+OGGIT_FORGEJO_PORT = 3100
 
 import sys
 import git
-import pkgutil
-import importlib
 import paramiko
 
 
@@ -1603,7 +1602,7 @@ def send_p2p():
             "success": False,
             "error": error_message
         }), 500
-    
+
     # Ejecutamos los scripts "runTorrentSeeder.py", que no reciben parámetros.
 
     journal.send("Running script 'runTorrentSeeder.py'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
@@ -1614,7 +1613,7 @@ def send_p2p():
     sleep(10)
     seeder_running = search_process('aria2c', f"{param_dict['name']}.img.torrent")  # El seeder se ejecuta con "aria2c" y el nombre de la imagen como parámetro
 
-    
+
     # Evaluamos las comprobaciones anteriores, para devolver la respuesta que corresponda:
     if seeder_running:
         journal.send("'runTorrentSeeder.py' results OK (ReturnCodes: None), and processes running", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
@@ -2220,6 +2219,48 @@ def rename_image():
 # -----------------------------------------------------------
 
 
+def git_forgejo_post_command(url, json_data):
+    token = ""
+    with open("/opt/opengnsys/etc/ogGitApiToken.cfg", "r", encoding='utf-8') as token_file:
+        token = token_file.read().strip()
+
+    if url[0] == "/":
+        url = url[1:]
+
+    url = f"http://localhost:{OGGIT_FORGEJO_PORT}/{url}"
+
+    journal.send(f"Executing forgejo POST to {url} with data {json_data}...", PRIORITY=journal.LOG_DEBUG, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
+
+    r = requests.post(
+        url,
+        json = json_data,
+        headers = { 'Authorization' : f"token {token}" },
+        timeout = 60
+    )
+
+    journal.send(f"Request status was {r.status_code}, content {r.content}", PRIORITY=journal.LOG_DEBUG, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
+
+    return r.status_code, r.content.decode('utf-8')
+
+def git_forgejo_create_repo(repo):
+    return git_forgejo_post_command("/api/v1/user/repos",
+                    {
+                        "auto_init" : False,
+                        "default_branch" : "main",
+                        "description" : "",
+                        "name" : repo,
+                        "private" : False
+                    })
+
+
+def git_forgejo_add_sshkey(pubkey, description = ""):
+    return git_forgejo_post_command("/api/v1/user/keys",
+                                    {
+                                        "key" : pubkey,
+                                        "read_only" : False,
+                                        "title" : description
+                                    })
+
 def git_compact_repository_task(repo, job_id):
     journal.send("Running function 'git_compact_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
 
@@ -2289,40 +2330,6 @@ def git_backup_repository_task(repo, params, job_id):
     journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
     recall_ogcore(data)
 
-
-def git_add_sshkey_task(oglive, description, job_id):
-    module = _load_installer()
-    print(f"Got {module}")
-    OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller')
-    installer = OpengnsysGitInstaller()
-
-    results = installer.add_ssh_key_from_squashfs(oglive_file = oglive)
-    keys_added = 0
-    keys_existed = 0
-    keys_failed = 0
-
-    for status, message in results:
-        if status == 200 or status == 201:
-            keys_added = keys_added + 1
-        elif status == 422:
-            keys_existed = keys_existed + 1
-        else:
-            keys_failed = keys_failed + 1
-            journal.send(f"Unrecognized reply from forgejo: code {status}, content {message}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
-
-
-    data = {
-        'job_id': job_id,
-        'keys_added' : keys_added,
-        'keys_failed' : keys_failed,
-        'keys_existed' : keys_existed,
-        'output' : message
-    }
-
-    journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
-    recall_ogcore(data)
-
-
 @app.route("/ogrepository/v1/git/repositories", methods=['GET'])
 def git_list_repositories():
     """
@@ -2369,42 +2376,6 @@ def git_list_repositories():
         "repositories": repos
     }), 200
 
-def _load_module(module_name):
-   # module = importlib.util.find_spec(module_name)
-    module = importlib.import_module(module_name)
-
-    if module is not None:
-        journal.send(f"Module {module_name} loaded successfully. Got {module}", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG")
-        return module
-
-    journal.send(f"Module {module_name} failed to load, not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="oggit-api_DEBUG")
-    return False
-
-def _load_installer():
-
-    journal.send(f"Loading oggit installer module", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG")
-
-    script_dir = os.path.dirname(os.path.realpath(__file__))
-
-    system_bin_path = '/opt/opengnsys/ogrepository/oggit/bin/'
-    system_lib_path = '/opt/opengnsys/ogrepository/oggit/lib/'
-    devel_lib_path = os.path.join(script_dir, "../../oggit/installer")
-
-    if devel_lib_path not in sys.path and os.path.isdir(devel_lib_path):
-        journal.send(f"Using {devel_lib_path} development library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG")
-        sys.path.append(devel_lib_path)
-
-    if system_bin_path not in sys.path:
-        journal.send(f"Using {system_bin_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG")
-        sys.path.append(system_bin_path)
-
-    if system_lib_path not in sys.path:
-        journal.send(f"Using {system_lib_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG")
-        sys.path.append(system_lib_path)
-
-    return _load_module("opengnsys_git_installer")
-
-
 @app.route("/ogrepository/v1/git/repositories", methods=['POST'])
 def git_create_repository():
     """
@@ -2434,11 +2405,7 @@ def git_create_repository():
         journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
         return jsonify({"status": "Repository already exists"}), 409
 
-    module = _load_installer()
-    print(f"Got {module}")
-    OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller')
-    installer = OpengnsysGitInstaller()
-    installer.add_forgejo_repo(repo)
+    ret = git_forgejo_create_repo(repo)
 
     journal.send(f"Repository {repo} created", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
 
@@ -2592,6 +2559,7 @@ def git_get_commits(repo, branch):
 
     max_commits = request.args.get('max_commits') or 100
     skip_commits = request.args.get('skip') or 0
+    filter_commits = (request.args.get('filter_commits') or 1) == 1
 
 
     git_repo = git.Repo(repo_path)
@@ -2613,14 +2581,41 @@ def git_get_commits(repo, branch):
         hash_to_tag[sha] = hash_to_tag[sha] + [ tag.name ]
 
 
+    first_commit = next(git_repo.iter_commits(reverse=True))
+
     commits = []
+    git_repo.iter_commits()
     for com in git_repo.iter_commits(branch, max_count = max_commits, skip = skip_commits):
         tag_list = []
 
+        # Initial check-ins contain only a .gitignore, used to test the repository is working,
+        # and do a force push to the destination.
+        #
+        # We don't want to show this one to the users by default, since it's not a functional
+        # restoration point.
+        #
+        # To make sure users can use git normally, we try to detect it in a slightly paranoid
+        # manner here. We want to avoid magic markers or to always skip the first commit, in
+        # case it might actually contain data.
+        skip_this_commit = False
+
+        # Check:
+        # * First commit, and we want filtering
+        # * There's only one blob
+        # * The one file is .gitignore.
+        if com == first_commit and filter_commits:
+            if len(com.tree.blobs) == 1:
+                for file in com.tree.traverse(depth=1, visit_once=False):
+                    if file.name == ".gitignore" and file.type == "blob":
+                        skip_this_commit = True
+
+        if skip_this_commit:
+            continue
+
+
         if com.hexsha in hash_to_tag:
             tag_list = hash_to_tag[com.hexsha]
 
-
         commits = commits + [
             {
                 "hexsha" : com.hexsha,
@@ -2879,49 +2874,33 @@ def git_add_sshkey():
     if not "description" in data:
         data["description"] = ""
 
-    if not ("ssh_key" in data or "oglive" in data):
-        journal.send(f"Can't add SSH keys, either ssh_key or oglive is required", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
+    if not ("ssh_key" in data ):
+        journal.send(f"Can't add SSH keys, ssh_key is required", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
         return jsonify({"error" : "Parameters missing, specify ssh_key or oglive"}), 400
 
+    status, content = git_forgejo_add_sshkey(data["ssh_key"], data["description"])
+    message = "Result unrecognized"
+    success = False
+    httpcode = 500
 
-
-    module = _load_installer()
-    print(f"Got {module}")
-    OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller')
-    installer = OpengnsysGitInstaller()
-
-    if "oglive" in data:
-        job_id = f"GitSshKey_{''.join(random.choice('0123456789abcdef') for char in range(8))}"
-        threading.Thread(target=git_add_sshkey_task, args=(data["oglive"], data["description"], job_id)).start()
-        return jsonify({
-            "success": True,
-            "output": "Extracting key from ogLive...",
-            "job_id": job_id
-        }), 200
+    if status == 200 or status == 201:
+        message = "SSH key added"
+        success = True
+        httpcode = 200
+    elif status == 422:
+        message = "SSH key already existed"
+        success = True
+        httpcode = 200
     else:
-        status, content = installer.add_forgejo_sshkey(data["ssh_key"], data["description"])
-        message = "Result unrecognized"
+        message = "Unrecognized reply from forgejo"
         success = False
-        httpcode = 500
+        httpcode = status
+        journal.send(f"Unrecognized reply from forgejo: code {status}, content {content}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
 
-        if status == 200 or status == 201:
-            message = "SSH key added"
-            success = True
-            httpcode = 200
-        elif status == 422:
-            message = "SSH key already existed"
-            success = True
-            httpcode = 200
-        else:
-            message = "Unrecognized reply from forgejo"
-            success = False
-            httpcode = status
-            journal.send(f"Unrecognized reply from forgejo: code {status}, content {content}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG")
-
-        return jsonify({
-            "success": success,
-            "output": message,
-        }), httpcode
+    return jsonify({
+        "success": success,
+        "output": message,
+    }), httpcode
 
 
 # --------------------------------------------------------------------------------------------

api/swagger.yaml

@@ -1982,9 +1982,6 @@ paths:
 
          Especificando **ssh_key** se agrega la cclave especificada directamente.
 
-         Especificando **oglive** se descarga si es necesario el .iso, se monta, y se extrae la clave.
-         Esta acción se hace en el fondo, y se devuelve un job_id.
-
       tags:
         - "Git"
       parameters:
@@ -2006,10 +2003,6 @@ paths:
                   type: string
                   example: "OgLive r20250518"
                   required: False
-                oglive:
-                  type: string
-                  example: "https://ognproject.evlt.uma.es/oglive/ogLive-noble-6.8.0-31-generic-amd64-r20250518.cf13a6d_20250519.iso"
-                  required: False
       responses:
         "200":
           description: "Exito"
@@ -2270,7 +2263,12 @@ paths:
     get:
       summary: "Obtener lista de commits en una rama"
       description: |
-        Devuelve una lista de commits de Git
+        Devuelve una lista de commits de Git.
+
+        **Filtrado de commits:**
+        Por defecto algunos commits que no son utiles para restaurar una imagen se ocultan.
+        Pasar filter_commits=0 para desactivar esta funcionalidad y ver todos los commits
+        del repositorio.
       tags:
         - "Git"
       parameters:
@@ -2294,6 +2292,11 @@ paths:
           required: false
           type: int
           description: "Commits a saltar (para paginación)"
+        - name: filter_commits
+          in: query
+          required: false
+          type: int
+          description: "Si aplicar filtrado de commits. Por defecto 1."
       responses:
         "200":
           description: "Lista de commits"

debian/changelog

@@ -1,3 +1,23 @@
+ogrepository (1.1.5) UNRELEASED; urgency=medium
+
+  * refs #2363 Fix breaking forgejo permissions
+
+ -- OpenGnsys <opengnsys@opengnsys.com>  Tue, 1 Jul 2025 16:05:12 +0000
+
+ogrepository (1.1.4) UNRELEASED; urgency=medium
+
+  * refs #2363 Reduce number of required dependencies
+
+ -- OpenGnsys <opengnsys@opengnsys.com>  Tue, 30 Jun 2025 15:50:11 +0000
+
+ogrepository (1.1.3) UNRELEASED; urgency=medium
+
+  * refs #2317 Fix performance problems, remove stats
+  * refs #2298 Add commmits endpoint
+
+ -- OpenGnsys <opengnsys@opengnsys.com>  Tue, 30 Jun 2025 15:50:11 +0000
+
+
 ogrepository (1.1.2) UNRELEASED; urgency=medium
 
   * Fix path handling when there are no repositories

debian/control

@@ -8,7 +8,7 @@ Build-Depends: debhelper-compat (= 12)
 Package: ogrepository
 Architecture: all
 Pre-Depends: debian-archive-keyring ,  debconf (>= 1.5.0),
-Depends: ${misc:Depends}, git, python3, python3-pip, python3-flask, python3-paramiko, python3-psutil, python3-flasgger,  samba, gunicorn, wakeonlan , lzop , partclone , qemu-utils , udpcast, uftp, mktorrent, aria2 , opengnsys-opentracker, opengnsys-gitinstaller, opengnsys-forgejo
+Depends: ${misc:Depends}, git, python3, python3-pip, python3-flask, python3-paramiko, python3-psutil, python3-flasgger,  samba, gunicorn, wakeonlan , lzop , partclone , qemu-utils , udpcast, uftp, mktorrent, aria2 , opengnsys-opentracker, opengnsys-forgejo, python3-git
 Description: Ogrepsoitory Package
  This package provides Ogrepository service.
 

debian/ogrepository.postinst

@@ -92,7 +92,7 @@ done
         (echo "$SAMBA_PASS"; echo "$SAMBA_PASS") | smbpasswd -a $SAMBA_USER
     fi
     systemctl enable ogrepo-api
-    # Configure Repo 
+    # Configure Repo
 
     cp /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg.tmpl /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg
     sed -i "s/SERVERIP/$OGREPO_IP/g" /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg
@@ -128,6 +128,18 @@ chown -R opengnsys:www-data /opt/opengnsys/ogrepository
 chmod 755 /opt/opengnsys
 chmod 755 /opt/opengnsys/ogrepository/bin/*
 
+# oggit -- these are from opengnsys-forgejo, and we're overwriting their
+# permissions above.
+#
+# Check if the paths exist just in case the opengnsys-forgejo package
+# stops being a dependency.
+if [ -d "/opt/opengnsys/ogrepository/var/lib/forgejo/" ] ; then
+    chown -R oggit:oggit /opt/opengnsys/ogrepository/var/lib/forgejo/
+fi
+
+if [ -d "/opt/opengnsys/ogrepository/oggit/" ] ; then
+    chown -R oggit:oggit /opt/opengnsys/ogrepository/oggit/
+fi
 
 # Install http server stuff
 # Reiniciar servicios si es necesario