From 88953696e74985b45b7143b15cc79233610a131f Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 28 Apr 2025 11:39:19 +0200 Subject: [PATCH 01/26] Git implementation and Swagger --- api/repo_api.py | 378 ++++++++++++++++++++++++++++++++++++++++++++--- api/swagger.yaml | 344 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 677 insertions(+), 45 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 22064e1..40c5ce5 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2,16 +2,16 @@ # -*- coding: utf-8 -*- """ - API de ogRepository, programada en Flask. + API de ogRepository, programada en Flask. Responde a peticiones HTTP (en principio, enviadas desde ogCore) mediante endpoints, que a su vez ejecutan los scripts Python almacenados en ogRepository. En ciertos casos, transforma los parámetros recibidos desde el portal, para adaptarlos a los que es necesario enviar a los scripts - (por ejemplo, a partir del ID de una imagen obtiene su nombre y su extensión). + (por ejemplo, a partir del ID de una imagen obtiene su nombre y su extensión). -Librerías Python requeridas: - flask (se puede instalar con "sudo apt install python3-flask") - - paramiko (se puede instalar con "sudo apt install python3-paramiko") +Librerías Python requeridas: - flask (se puede instalar con "sudo apt install python3-flask") + - paramiko (se puede instalar con "sudo apt install python3-paramiko") - requests (se puede instalar con "sudo apt install python3-requests") - No es necesario instalarlo en Ubuntu 24 - - flasgger (se puede instalar con "sudo apt install python3-flasgger") + - flasgger (se puede instalar con "sudo apt install python3-flasgger") """ # -------------------------------------------------------------------------------------------- @@ -48,6 +48,19 @@ trash_file = '/opt/opengnsys/ogrepository/etc/trashinfo.json' config_file = '/opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg' + + +# -------------------------------------------------------------------------------------------- +# GIT +# -------------------------------------------------------------------------------------------- + +REPOSITORIES_BASE_PATH = "/opt/opengnsys/ogrepository/oggit/git/oggit/" + +import git +import pkgutil +import importlib + + # -------------------------------------------------------------------------------------------- # FUNCTIONS # -------------------------------------------------------------------------------------------- @@ -73,9 +86,9 @@ swagger = Swagger(app, template=swagger_template) def get_IPcore(): """ Obtiene el valor asociado a la variable "IPcore", desde el archivo '/opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg'. Retorna la IP encontrada (que corresponde a la IP de ogCore), o un error (si no la encuentra). - """ + """ journal.send("Running function 'get_IPcore'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - + IPcore = None with open(config_file, 'r') as file: for line in file: @@ -327,7 +340,7 @@ def check_remote_backup(image_name, remote_ip, remote_user, remote_path, job_id) # Creamos un bucle infinito: while True: # Comprobamos si ya se ha copiado la imagen y cada uno de sus archivos asociados (y almacenamos el resultado): - try: + try: for ext in extensions: sftp_client.stat(f"{remote_path}{image_name}{ext}") all_files_copied = True @@ -601,7 +614,7 @@ def recall_ogcore(data): def check_file_exists(file_path): """ Comprueba la existencia del archivo cuya ruta recibe como parámetro. - Si el archivo existe devuelve "True", y si no devuelve "False". + Si el archivo existe devuelve "True", y si no devuelve "False". """ journal.send("Running function 'check_file_exists'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -671,7 +684,7 @@ def get_repo_status(): journal.send("Running script 'getRepoStatus.py'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") # Ejecutamos el script "getRepoStatus.py", y almacenamos el resultado: result = subprocess.run(['python3', f"{script_path}/getRepoStatus.py"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='UTF8') - + # Evaluamos el resultado de la ejecución, y devolvemos la respuesta: if result.returncode == 0: journal.send("Script 'getRepoStatus.py' result OK (ReturnCode: 0)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -973,7 +986,7 @@ def recover_image(): # Almacenamos el parámetro "ID_img" (enviado por JSON): json_data = json.loads(request.data) - image_id = json_data.get("ID_img") + image_id = json_data.get("ID_img") # Obtenemos el nombre y la extensión de la imagen: param_dict = get_image_params(image_id, "trash") @@ -1136,7 +1149,7 @@ def import_image(): if result.returncode is None: journal.send("Script 'importImage.py' result OK (ReturnCode: None)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") journal.send("{'component':'ogRepo', 'severity':'INFO', 'http_code':'200', 'operation':'Run script importImage.py', 'desc':'Result OK (ReturnCode: None)'}", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api") - + # Si el resultado es correcto, llamamos a la función "check_lock_local" en un hilo paralelo # (para que compruebe si la imagen se ha acabado de importar exitosamente): journal.send("Calling function 'check_lock_local'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -1179,7 +1192,7 @@ def import_image(): def backup_image(): """ Este endpoint exporta la imagen especificada como primer parámetro (y todos sus archivos asociados), desde el servidor local a un equipo remoto (que no tiene por qué ser un repositorio). Para ello, ejecuta el script "backupImage.py", con el nombre de la imagen como primer parámetro, la IP o hostname del equipo remoto como segundo parámetro, - el usuario con el que conectar al equipo remoto como tercer parámetro, y la ruta remota en la que copiar la imagen como cuarto parámetro. + el usuario con el que conectar al equipo remoto como tercer parámetro, y la ruta remota en la que copiar la imagen como cuarto parámetro. """ journal.send("Running endpoint 'Hacer backup de una Imagen'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -1260,7 +1273,7 @@ def backup_image(): }), 200 else: journal.send("Script 'backupImage.py' result KO (Backup image failed)", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - journal.send("{'component':'ogRepo', 'severity':'ERROR', 'http_code':'500', 'operation':'Run script backupImage.py', 'desc':'Result KO (Backup image failed)'}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api") + journal.send("{'component':'ogRepo', 'severity':'ERROR', 'http_code':'500', 'operation':'Run script backupImage.py', 'desc':'Result KO (Backup image failed)'}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api") return jsonify({ "success": False, "error": "Backup image failed" @@ -1616,7 +1629,7 @@ def send_p2p(): "success": False, "error": error_message }), 500 - + # --------------------------------------------------------- @@ -1884,7 +1897,7 @@ def stop_p2p(): journal.send("Running script 'stopP2P.py'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") # Ejecutamos el script "stopP2P.py", y almacenamos el resultado (este script si que requiere ser ejecutado con "sudo"): result = subprocess.run(['sudo', 'python3', f"{script_path}/stopP2P.py"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='UTF8') - + # Evaluamos el resultado de la ejecución, y devolvemos la respuesta: if result.returncode == 0: journal.send("Script 'stopP2P.py' result OK (ReturnCode: 0)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -1918,7 +1931,7 @@ def convert_virtual_image(): """ Este endpoint convierte la imagen virtual especificada como parámetro en una imagen "img" como las que se generan desde OpenGnsys (con "partclone" y "lzop"), por lo que luego puede ser restaurada como cualquier otra imagen del repositorio. Para ello, ejecuta el script "convertVMtoIMG.py", con el nombre de la imagen virtual como primer parámetro, - y el sistema de archivos de la partición a clonar (en formato "blkid") como segundo parámetro. + y el sistema de archivos de la partición a clonar (en formato "blkid") como segundo parámetro. """ journal.send("Running endpoint 'Convertir imagen virtual a imagen OpenGnsys'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -1981,7 +1994,7 @@ def convert_virtual_image(): if result.returncode is None: journal.send("Script 'convertVMtoIMG.py' result OK (ReturnCode: None)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") journal.send("{'component':'ogRepo', 'severity':'INFO', 'http_code':'200', 'operation':'Run script convertVMtoIMG.py', 'desc':'Result OK (ReturnCode: None)'}", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api") - + # Si el resultado es correcto, llamamos a la función "check_virtual_image_conversion" en un hilo paralelo # (para que compruebe si la imagen se ha acabado de convertir exitosamente): journal.send("Calling function 'check_virtual_image_conversion'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2024,7 +2037,7 @@ def convert_virtual_image(): def convert_image_to_virtual(): """ Este endpoint convierte la imagen de OpenGnsys especificada como parámetro en una imagen virtual con la extensión especificada (".vdi", ".vmdk", etc). Para ello, ejecuta el script "convertIMGtoVM.py", con el nombre de la imagen "img" como primer parámetro, - y la extensión del disco virtual destino (sin punto) como segundo parámetro. + y la extensión del disco virtual destino (sin punto) como segundo parámetro. """ journal.send("Running endpoint 'Convertir imagen OpenGnsys a imagen virtual'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2086,7 +2099,7 @@ def convert_image_to_virtual(): if result.returncode is None: journal.send("Script 'convertIMGtoVM.py' result OK (ReturnCode: None)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") journal.send("{'component':'ogRepo', 'severity':'INFO', 'http_code':'200', 'operation':'Run script convertIMGtoVM.py', 'desc':'Result OK (ReturnCode: None)'}", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api") - + # Si el resultado es correcto, llamamos a la función "check_virtual_image_reconversion" en un hilo paralelo # (para que compruebe si la imagen se ha acabado de convertir exitosamente): journal.send("Calling function 'check_virtual_image_reconversion'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2128,7 +2141,7 @@ def convert_image_to_virtual(): @app.route("/ogrepository/v1/images/rename", methods=['PUT']) def rename_image(): """ Este endpoint renombra la imagen especificada como primer parámetro (y todos sus archivos asociados), asignando el nombre especificado como segundo parámetro. - Para ello, ejecuta el script "renameImage.py", con el nombre original de la imagen como primer parámetro, y el nuevo nombre a asignar como segundo parámetro. + Para ello, ejecuta el script "renameImage.py", con el nombre original de la imagen como primer parámetro, y el nuevo nombre a asignar como segundo parámetro. """ journal.send("Running endpoint 'Renombrar una Imagen'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2194,6 +2207,329 @@ def rename_image(): +# ----------------------------------------------------------- +# ____ _ _ +# / ___(_) |_ +# | | _| | __| +# | |_| | | |_ +# \____|_|\__| +# +# ----------------------------------------------------------- + + +@app.route("/ogrepository/v1/git/repositories", methods=['GET']) +def git_list_repositories(): + """ + Retrieve a list of Git repositories. + + This endpoint scans the OpenGnsys image path for directories that + appear to be Git repositories (i.e., they contain a "HEAD" file). + It returns a JSON response containing the names of these repositories. + + Returns: + Response: A JSON response with a list of repository names or an + error message if the repository storage is not found. + - 200 OK: When the repositories are successfully retrieved. + - 500 Internal Server Error: When the repository storage is not found. + + Example JSON response: + { + "repositories": ["repo1", "repo2"] + } + """ + journal.send("Running endpoint 'Obtener Información de todos los repositorios Git'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + if not os.path.isdir(REPOSITORIES_BASE_PATH): + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 + + repos = [] + for entry in os.scandir(REPOSITORIES_BASE_PATH): + if entry.is_dir(follow_symlinks=False) and os.path.isfile(os.path.join(entry.path, "HEAD")): + name = entry.name + if name.endswith(".git"): + name = name[:-4] + + repos = repos + [name] + + + journal.send(f"Returning {len(repos)} repositories", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + return jsonify({ + "repositories": repos + }), 200 + +def _load_module(module_name): + import importlib + return importlib.util.find_spec(module_name) is not None + return False + +def _load_installer(): + return _load_module("opengnsys-git-installer") + + +@app.route("/ogrepository/v1/git/repositories", methods=['POST']) +def git_create_repository(): + """ + Create a new Git repository. + + This endpoint creates a new Git repository with the specified name. + If the repository already exists, it returns a status message indicating so. + + Args: + repo (str): The name of the repository to be created. + + Returns: + Response: A JSON response with a status message and HTTP status code. + - 200: If the repository already exists. + - 201: If the repository is successfully created. + """ + data = request.json + + if data is None: + journal.send(f"Can't create repository, JSON post data missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + repo = data["name"] + + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if os.path.isdir(repo_path): + journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"status": "Repository already exists"}), 200 + + _import_installer() + installer = OpengnsysGitInstaller() + installer.add_forgejo_repo(repo) + + journal.send(f"Repository {repo} created", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + return jsonify({"status": "Repository created"}), 201 + +@app.route("/ogrepository/v1/git/repositories", methods=['DELETE']) +def git_delete_repository(): + return jsonify({"error" : "Not implemented"}), 500 + +@app.route("/ogrepository/v1/git/repositories//sync", methods=['POST']) +def git_sync_repository(repo): + return jsonify({"error" : "Not implemented"}), 500 + +@app.route("/ogrepository/v1/git/repositories//backup", methods=['POST']) +def git_backup_repository(repo): + return jsonify({"error" : "Not implemented"}), 500 + +@app.route("/ogrepository/v1/git/repositories//compact", methods=['POST']) +def git_compact_repository(repo): + return jsonify({"error" : "Not implemented"}), 500 + +@app.route("/ogrepository/v1/git/repositories//branches", methods=['GET']) +def git_get_branches(repo): + """ + Retrieve the list of branches for a given repository. + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a list of branch names or an error message if the repository is not found. + - 200: A JSON object with a "branches" key containing a list of branch names. + - 404: A JSON object with an "error" key containing the message "Repository not found" if the repository does not exist. + """ + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository not found"}), 404 + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + + branches = [] + for branch in git_repo.branches: + branches = branches + [branch.name] + + journal.send(f"Returning {len(branches)} branches", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return { + "branches": branches + } + +@app.route("/ogrepository/v1/git/repositories//branches/", methods=['POST']) +def git_create_branch(repo, branch): + """Create a given branch in a given repository + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a creation status + - 201: A JSON object with a "status" key containing "created" + - 404: A JSON object with an "error" key containing the message "Repository not found" + - 409: A JSON object with an "error" key containing the message "Tag already exists" + """ + + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't create branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository not found"}), 404 + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + data = request.json + if data is None: + journal.send(f"Can't create branch. JSON post data missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "commit" in data: + journal.send(f"Can't create branch. Commit parameter missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "commit parameter missing"}), 400 + + if branch in git_repo.branches: + journal.send(f"Can't create branch. Already found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Branch already exists"}), 409 + + git_repo.create_tag(branch, ref = data["commit"]) + + journal.send(f"Branch {branch} created in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"status": "created"}), 201 + +@app.route("/ogrepository/v1/git/repositories//branches/", methods=['DELETE']) +def git_delete_branch(repo, branch): + """Delete a given branch in a given repository + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a list of branch names or an error message if the repository is not found. + - 200: A JSON object with a "status" key containing "deleted" + - 404: A JSON object with an "error" key containing the message "Repository not found" or "Branch not found" + """ + + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't delete branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return {"error": "Repository not found"}, 404 + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + + if not branch in git_repo.branches: + journal.send(f"Can't delete branch. Not found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Branch not found"}), 404 + + git_repo.delete_head(branch) + journal.send(f"Branch {branch} deleted in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + return jsonify({"status": "deleted"}), 200 + + +@app.route("/ogrepository/v1/git/repositories//tags", methods=['GET']) +def git_list_tags(repo): + """ + Retrieve the list of tags for a given repository. + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a list of tag names or an error message if the repository is not found. + - 200: A JSON object with a "branches" key containing a list of tag names. + - 404: A JSON object with an "error" key containing the message "Repository not found" if the repository does not exist. + """ + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository not found"}), 404 + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + + tags = [] + for tag in git_repo.tags: + tags = tags + [tag.name] + + journal.send(f"Returning {len(tags)} branches", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return { + "tags": tags + } + + +@app.route("/ogrepository/v1/git/repositories//tags/", methods=['POST']) +def git_create_tag(repo, tag): + """Create a given tag in a given repository + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a creation status + - 200: A JSON object with a "status" key containing "created" + - 404: A JSON object with an "error" key containing the message "Repository not found" + - 409: A JSON object with an "error" key containing the message "Tag already exists" + """ + + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't create tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository not found"}), 404 + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + data = request.json + if data is None: + journal.send(f"Can't create tag. JSON post data missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "commit" in data: + journal.send(f"Can't create tag. Commit parameter missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "commit parameter missing"}), 400 + + if tag in git_repo.tags: + journal.send(f"Can't create tag. Already found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Tag already exists"}), 409 + + git_repo.create_tag(tag, ref = data["commit"]) + + journal.send(f"Tag {tag} created in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"status": "created"}), 200 + +@app.route("/ogrepository/v1/git/repositories//tags/", methods=['DELETE']) +def git_delete_tag(repo, tag): + """Delete a given tag in a given repository + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a list of tag names or an error message if the repository is not found. + - 200: A JSON object with a "status" key containing "deleted" + - 404: A JSON object with an "error" key containing the message "Repository not found" or "Tag not found" + """ + + repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't delete tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return {"error": "Repository not found"}, 404 + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + + if not tag in git_repo.tags: + journal.send(f"Can't delete tag. Not found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Tag not found"}), 404 + + git_repo.delete_head(tag) + journal.send(f"Tag {tag} deleted in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + return jsonify({"status": "deleted"}), 200 + + + + # -------------------------------------------------------------------------------------------- diff --git a/api/swagger.yaml b/api/swagger.yaml index a7a8ebc..a7ca255 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -4,7 +4,7 @@ info: version: "1.0" description: | --- - + # ----------------------------------------------------------------------------------------------------------- # Esto hace que el Swagger se ordene por los tags (apartados), de la forma especificada: @@ -18,7 +18,7 @@ tags: - name: "Transferencia entre Repositorios y Backup" - name: "Importar y Exportar Máquinas Virtuales" - name: "Varios" - + - name: "Git" # ----------------------------------------------------------------------------------------------------------- # Apartado "Estado de ogRepository" @@ -30,7 +30,7 @@ paths: summary: "Obtener Información de Estado de ogRepository" description: > Este endpoint ejecuta el script "**getRepoStatus.py**" y devuelve su salida en formato JSON, - incluyendo información sobre la CPU, memoria RAM, disco duro, servicios, y procesos específicos de ogRepository, e instalación de ogGit. + incluyendo información sobre la CPU, memoria RAM, disco duro, servicios, y procesos específicos de ogRepository, e instalación de ogGit. tags: - "Estado de ogRepository" responses: @@ -140,14 +140,14 @@ paths: # ----------------------------------------------------------------------------------------------------------- # Apartado "Información de Imágenes" # ----------------------------------------------------------------------------------------------------------- - + /ogrepository/v1/images: put: summary: "Actualizar Información del Repositorio" description: | Este endpoint actualiza la información de las imágenes almacenadas en el repositorio, reflejándola en los archivos "**repoinfo.json**" y "**trashinfo.json**". Utiliza el script "**updateRepoInfo.py**", que a su vez llama al script "**updateTrashInfo.py**", para actualizar también la información de la papelera. - + No hace falta que se le llame al crear o exportar una imagen, ya que lo llama el endpoint "**Crear archivos auxiliares**" (que sí debe ser llamado en esos casos). tags: - "Información de Imágenes" @@ -192,7 +192,7 @@ paths: get: summary: "Obtener Información de todas las Imágenes" description: | - Este endpoint ejecuta el script "**getRepoInfo.py**" con el parámetro "**all**", para devolver información de todas las imágenes almacenadas en el repositorio y en la papelera, que a su vez llama al script "**updateRepoInfo.py**", para actualizar previamente la información del repositorio. + Este endpoint ejecuta el script "**getRepoInfo.py**" con el parámetro "**all**", para devolver información de todas las imágenes almacenadas en el repositorio y en la papelera, que a su vez llama al script "**updateRepoInfo.py**", para actualizar previamente la información del repositorio. Devuelve detalles como el nombre de la imagen, tipo, nombre del cliente, clonador, compresor, sistema de archivos, tamaño de los datos, tamaño de la imagen, y hashes MD5. tags: - "Información de Imágenes" @@ -1257,7 +1257,7 @@ paths: # ----------------------------------------------------------------------------------------------------------- - #/ogrepository/v1/p2p: + #/ogrepository/v1/p2p: delete: summary: "Cancelar Transmisiones P2P" description: | @@ -1311,10 +1311,10 @@ paths: Este endpoint importa la imagen especificada desde un repositorio remoto al repositorio local (en el que se ejecuta el endpoint). Utiliza el script "**importImage.py**", que recibe como parámetros el nombre de la imagen, la IP o hostname del servidor remoto, y el usuario con el que conectar al servidor remoto. que a su vez llama al script "**updateRepoInfo.py**", para actualizar la información del repositorio. - + **NOTA**: Este endpoint es asíncrono, ya que puede tardar mucho tiempo, por lo que solo informa de que la imagen se está importando, y abre un proceso paralelo, que avisará a ogCore cuando finalice la tarea (llamando a un endpoint de ogCore). tags: - - "Transferencia entre Repositorios y Backup" + - "Transferencia entre Repositorios y Backup" parameters: - name: JSON in: body @@ -1436,7 +1436,7 @@ paths: * **ID_img** - Identificador de la imagen, correspondiente al contenido del archivo 'full.sum' * **repo_ip** - Dirección IP del servidor remoto * **user** - Usuario con el que conectar al servidor remoto - * **remote_path** - Ruta remota en la que copiar la imagen + * **remote_path** - Ruta remota en la que copiar la imagen schema: type: object properties: @@ -1457,7 +1457,7 @@ paths: example: "/home/opengnsys" responses: "200": - description: "Se está haciendo backup de la imagen." + description: "Se está haciendo backup de la imagen." schema: type: object properties: @@ -1532,20 +1532,20 @@ paths: post: summary: "Convertir Imagen Virtual a Imagen OpenGnsys" description: | - Este endpoint convierte la imagen virtual especificada como primer parámetro en una imagen "img" como las que se generan desde OpenGnsys, debiendo haberse copiado previamente en la ruta "opt/opengnsys/ogrepository/images_virtual". - Utiliza el script "**convertVMtoIMG.py**", que recibe como parámetros el nombre de la imagen virtual, y el sistema de archivos de la partición a clonar (en formato "blkid"). - Se puede comprobar todos los sistemas de archivos aceptados por "blkid" ejecutando el comando "blkid -k". - + Este endpoint convierte la imagen virtual especificada como primer parámetro en una imagen "img" como las que se generan desde OpenGnsys, debiendo haberse copiado previamente en la ruta "opt/opengnsys/ogrepository/images_virtual". + Utiliza el script "**convertVMtoIMG.py**", que recibe como parámetros el nombre de la imagen virtual, y el sistema de archivos de la partición a clonar (en formato "blkid"). + Se puede comprobar todos los sistemas de archivos aceptados por "blkid" ejecutando el comando "blkid -k". + **NOTA**: Este endpoint es asíncrono, ya que puede tardar mucho tiempo, por lo que solo informa de que la imagen virtual se está convirtiendo, y abre un proceso paralelo, que avisará a ogCore cuando finalice la tarea (llamando a un endpoint de ogCore). tags: - - "Importar y Exportar Máquinas Virtuales" + - "Importar y Exportar Máquinas Virtuales" parameters: - name: JSON in: body required: true description: | - * **virtual_image** - Nombre de la imagen virtual, con extensión - * **filesystem** - Sistema de archivos de la partición a clonar, en formato "blkid" + * **virtual_image** - Nombre de la imagen virtual, con extensión + * **filesystem** - Sistema de archivos de la partición a clonar, en formato "blkid" schema: type: object properties: @@ -1640,19 +1640,19 @@ paths: put: summary: "Convertir Imagen OpenGnsys a Imagen Virtual" description: | - Este endpoint convierte la imagen "img" especificada como primer parámetro en una imagen virtual con la extensión especificada como segundo parámetro ("vdi", "vmdk", etc), guardándola en la ruta "opt/opengnsys/ogrepository/images_virtual/export". - Utiliza el script "**convertIMGtoVM.py**", que recibe como parámetros el nombre de la imagen, y la extensión del disco virtual destino ("vdi", "vmdk", etc). - + Este endpoint convierte la imagen "img" especificada como primer parámetro en una imagen virtual con la extensión especificada como segundo parámetro ("vdi", "vmdk", etc), guardándola en la ruta "opt/opengnsys/ogrepository/images_virtual/export". + Utiliza el script "**convertIMGtoVM.py**", que recibe como parámetros el nombre de la imagen, y la extensión del disco virtual destino ("vdi", "vmdk", etc). + **NOTA**: Este endpoint es asíncrono, ya que puede tardar mucho tiempo, por lo que solo informa de que la imagen se está convirtiendo a virtual, y abre un proceso paralelo, que avisará a ogCore cuando finalice la tarea (llamando a un endpoint de ogCore). tags: - - "Importar y Exportar Máquinas Virtuales" + - "Importar y Exportar Máquinas Virtuales" parameters: - name: JSON in: body required: true description: | * **ID_img** - Identificador de la imagen, correspondiente al contenido del archivo 'full.sum' - * **vm_extension** - Extensión del disco virtual destino ("vdi", "vmdk", etc) + * **vm_extension** - Extensión del disco virtual destino ("vdi", "vmdk", etc) schema: type: object properties: @@ -1883,7 +1883,7 @@ paths: summary: "Renombrar una Imagen" description: | Este endpoint renombra la imagen especificada como primer parámetro (y todos sus archivos asociados), asignando el nombre especificado como segundo parámetro. - Utiliza el script "**renameImage.py**", que recibe como parámetros el nombre original y el nuevo nombre a asignar. + Utiliza el script "**renameImage.py**", que recibe como parámetros el nombre original y el nuevo nombre a asignar. tags: - "Varios" parameters: @@ -1960,3 +1960,299 @@ paths: example: "(Exception description)" # ----------------------------------------------------------------------------------------------------------- + + +# ----------------------------------------------------------- +# ____ _ _ +# / ___(_) |_ +# | | _| | __| +# | |_| | | |_ +# \____|_|\__| +# +# ----------------------------------------------------------- + + /ogrepository/v1/git/repositories: + get: + summary: "Obtener lista de repositorios" + description: | + Devuelve una lista de repositorios de Git + tags: + - "Git" + parameters: + responses: + "200": + description: "Lista de repositorios" + schema: + type: array + items: + type: string + example: linux + + "500": + description: "Excepción" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + post: + summary: "Crear repositorio" + description: | + Crea un repositorio nuevo de Git + tags: + - "Git" + parameters: + - name: JSON + in: body + required: true + description: | + * **name** - Nombre de repositorio + schema: + type: object + properties: + name: + type: string + example: linux + responses: + "201": + description: "Repositorio creado" + schema: + type: object + properties: + status: + type: string + example: "Repository created" + "500 (Exception)": + description: "JSON post data missing" + schema: + type: object + properties: + error: + type: string + example: "Parameters missing" + /ogrepository/v1/git/tags: + get: + summary: "Obtener lista de tags" + description: | + Devuelve una lista de tags de Git + tags: + - "Git" + parameters: + responses: + "200": + description: "Lista de tags" + schema: + type: array + items: + type: string + example: v0.1 + + "500": + description: "Excepción" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + /ogrepository/v1/git/tags/{branchId}: + post: + summary: "Crear tag" + description: | + Crea una tag de git + tags: + - "Git" + parameters: + - name: JSON + in: body + required: true + description: | + * **commit** - Commit al que apunta el tag nuevo. Puede ser un nombre de otra rama/tag. + schema: + type: object + properties: + name: + type: string + example: HEAD + responses: + "201": + description: "Tag creado" + schema: + type: object + properties: + status: + type: string + example: created + + "500": + description: "Excepción" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + "404": + description: "El repositorio no existe" + schema: + type: object + properties: + error: + type: string + example: "Repository not found" + "409": + description: "El tag ya existe" + schema: + type: object + properties: + error: + type: string + example: "Tag already exists" + delete: + summary: "Eliminar tag" + description: | + Elimina un tag de git + tags: + - "Git" + parameters: + responses: + "200": + description: "Tag eliminado" + schema: + type: object + properties: + status: + type: string + example: deleted + "500": + description: "Git no instalado" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + /ogrepository/v1/git/branches: + get: + summary: "Obtener lista de branches" + description: | + Devuelve una lista de branches de Git + tags: + - "Git" + parameters: + responses: + "200": + description: "Lista de branches" + schema: + type: array + items: + type: string + example: main + + "500": + description: "Excepción" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + "404": + description: "El repositorio no existe" + schema: + type: object + properties: + error: + type: string + example: "Repository not found" + "409": + description: "El branch ya existe" + schema: + type: object + properties: + error: + type: string + example: "Branch already exists" + + /ogrepository/v1/git/branches/{branchId}: + post: + summary: "Crear branch" + description: | + Crea una rama de git + tags: + - "Git" + parameters: + - name: JSON + in: body + required: true + description: | + * **commit** - Commit al que apunta la rama nueva. Puede ser un nombre de otra rama/tag. + schema: + type: object + properties: + name: + type: string + example: HEAD + responses: + "201": + description: "Rama creada" + schema: + type: array + items: + type: string + example: main + + "500": + description: "Git no instalado" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + delete: + summary: "Eliminar branch" + description: | + Elimina una rama de git + tags: + - "Git" + parameters: + responses: + "200": + description: "Branch eliminado" + schema: + type: object + properties: + status: + type: string + example: deleted + "500": + description: "Excepción" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" \ No newline at end of file From 74efebf3c8a98d541be7533220483b26c4c9f2ae Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 28 Apr 2025 12:14:36 +0200 Subject: [PATCH 02/26] Swagger fixes --- api/swagger.yaml | 94 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index a7ca255..1508c93 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1978,15 +1978,17 @@ paths: Devuelve una lista de repositorios de Git tags: - "Git" - parameters: responses: "200": description: "Lista de repositorios" schema: - type: array - items: - type: string - example: linux + type: object + properties: + repositories: + type: array + items: + type: string + example: linux "500": description: "Excepción" @@ -2034,7 +2036,7 @@ paths: error: type: string example: "Parameters missing" - /ogrepository/v1/git/tags: + /ogrepository/v1/git/repositories/{repository}/tags: get: summary: "Obtener lista de tags" description: | @@ -2042,14 +2044,22 @@ paths: tags: - "Git" parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" responses: "200": description: "Lista de tags" schema: - type: array - items: - type: string - example: v0.1 + type: object + properties: + tags: + type: array + items: + type: string + example: v0.1 "500": description: "Excepción" @@ -2062,7 +2072,7 @@ paths: exception: type: string example: "(Exception description)" - /ogrepository/v1/git/tags/{branchId}: + /ogrepository/v1/git/repositories/{repository}/tags/{tag}: post: summary: "Crear tag" description: | @@ -2070,6 +2080,16 @@ paths: tags: - "Git" parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" + - name: tag + in: path + required: true + type: string + description: "Tag del repositorio" - name: JSON in: body required: true @@ -2125,6 +2145,16 @@ paths: tags: - "Git" parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" + - name: tag + in: path + required: true + type: string + description: "Rama del repositorio" responses: "200": description: "Tag eliminado" @@ -2135,7 +2165,7 @@ paths: type: string example: deleted "500": - description: "Git no instalado" + description: "Excepción" schema: type: object properties: @@ -2145,7 +2175,7 @@ paths: exception: type: string example: "(Exception description)" - /ogrepository/v1/git/branches: + /ogrepository/v1/git/repositories/{repository}/branches: get: summary: "Obtener lista de branches" description: | @@ -2153,14 +2183,22 @@ paths: tags: - "Git" parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" responses: "200": description: "Lista de branches" schema: - type: array - items: - type: string - example: main + type: object + properties: + branches: + type: array + items: + type: string + example: main "500": description: "Excepción" @@ -2190,7 +2228,7 @@ paths: type: string example: "Branch already exists" - /ogrepository/v1/git/branches/{branchId}: + /ogrepository/v1/git/repositories/{repository}/branches/{branch}: post: summary: "Crear branch" description: | @@ -2198,6 +2236,16 @@ paths: tags: - "Git" parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" + - name: branch + in: path + required: true + type: string + description: "Branch del repositorio" - name: JSON in: body required: true @@ -2236,6 +2284,16 @@ paths: tags: - "Git" parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" + - name: branch + in: path + required: true + type: string + description: "Branch del repositorio" responses: "200": description: "Branch eliminado" From ba5384ea776ca8658bfd5cd5cca1fb40d25a9c05 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Tue, 6 May 2025 09:59:44 +0200 Subject: [PATCH 03/26] Add git repository GC function --- api/repo_api.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/api/repo_api.py b/api/repo_api.py index 40c5ce5..8c78003 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2217,6 +2217,24 @@ def rename_image(): # ----------------------------------------------------------- +def git_compact_repository_task(repo, job_id): + journal.send("Running function 'git_compact_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + git_repo = git.Repo(git_repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', git_repo_path) + + git_repo.git.gc() + + data = { + 'job_id': job_id, + 'success': True + } + + journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + recall_ogcore(data) + + @app.route("/ogrepository/v1/git/repositories", methods=['GET']) def git_list_repositories(): """ @@ -2319,7 +2337,22 @@ def git_backup_repository(repo): @app.route("/ogrepository/v1/git/repositories//compact", methods=['POST']) def git_compact_repository(repo): - return jsonify({"error" : "Not implemented"}), 500 + journal.send("Running endpoint 'Compactar repositorio Git'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + if not os.path.isdir(REPOSITORIES_BASE_PATH): + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 + + + job_id = f"GitGC_{''.join(random.choice('0123456789abcdef') for char in range(8))}" + threading.Thread(target=git_compact_repository_task, args=(repo, job_id,)).start() + + return jsonify({ + "success": True, + "output": "Compacting...", + "job_id": job_id + }), 200 + @app.route("/ogrepository/v1/git/repositories//branches", methods=['GET']) def git_get_branches(repo): From 9a9cf17403ffb37ad934fffc0dd276bc0339e8f8 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 7 May 2025 09:18:01 +0200 Subject: [PATCH 04/26] Fix installer module loading --- api/repo_api.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 8c78003..68b79c8 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -56,6 +56,7 @@ config_file = '/opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg' REPOSITORIES_BASE_PATH = "/opt/opengnsys/ogrepository/oggit/git/oggit/" +import sys import git import pkgutil import importlib @@ -2278,12 +2279,34 @@ def git_list_repositories(): }), 200 def _load_module(module_name): - import importlib - return importlib.util.find_spec(module_name) is not None + # module = importlib.util.find_spec(module_name) + module = importlib.import_module(module_name) + + if module is not None: + journal.send(f"Module {module_name} loaded successfully. Got {module}", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") + return module + + journal.send(f"Module {module_name} failed to load, not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="oggit-api_DEBUG") return False def _load_installer(): - return _load_module("opengnsys-git-installer") + + journal.send(f"Loading oggit installer module", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") + + script_dir = os.path.dirname(os.path.realpath(__file__)) + + system_lib_path = '/opt/opengnsys/oggit/bin/' + devel_lib_path = os.path.join(script_dir, "../../oggit/installer") + + if devel_lib_path not in sys.path and os.path.isdir(devel_lib_path): + journal.send(f"Using {devel_lib_path} development library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") + sys.path.append(devel_lib_path) + + if system_lib_path not in sys.path: + journal.send(f"Using {system_lib_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") + sys.path.append(system_lib_path) + + return _load_module("opengnsys_git_installer") @app.route("/ogrepository/v1/git/repositories", methods=['POST']) @@ -2305,17 +2328,19 @@ def git_create_repository(): data = request.json if data is None: - journal.send(f"Can't create repository, JSON post data missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create repository, JSON post data missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "Parameters missing"}), 400 repo = data["name"] repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if os.path.isdir(repo_path): - journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "Repository already exists"}), 200 - _import_installer() + module = _load_installer() + print(f"Got {module}") + OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller') installer = OpengnsysGitInstaller() installer.add_forgejo_repo(repo) From a387af27d0295471035abd6776021199cfbfda60 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 7 May 2025 09:18:13 +0200 Subject: [PATCH 05/26] Fix error logging, wrong constant --- api/repo_api.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 68b79c8..0df2ff9 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2259,7 +2259,7 @@ def git_list_repositories(): journal.send("Running endpoint 'Obtener Información de todos los repositorios Git'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") if not os.path.isdir(REPOSITORIES_BASE_PATH): - journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 repos = [] @@ -2365,7 +2365,7 @@ def git_compact_repository(repo): journal.send("Running endpoint 'Compactar repositorio Git'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") if not os.path.isdir(REPOSITORIES_BASE_PATH): - journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 @@ -2394,7 +2394,7 @@ def git_get_branches(repo): """ repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if not os.path.isdir(repo_path): - journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 git_repo = git.Repo(repo_path) @@ -2426,7 +2426,7 @@ def git_create_branch(repo, branch): repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if not os.path.isdir(repo_path): - journal.send(f"Can't create branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 git_repo = git.Repo(repo_path) @@ -2434,20 +2434,20 @@ def git_create_branch(repo, branch): data = request.json if data is None: - journal.send(f"Can't create branch. JSON post data missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create branch. JSON post data missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "Parameters missing"}), 400 if not "commit" in data: - journal.send(f"Can't create branch. Commit parameter missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create branch. Commit parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "commit parameter missing"}), 400 if branch in git_repo.branches: - journal.send(f"Can't create branch. Already found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create branch. Already found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Branch already exists"}), 409 git_repo.create_tag(branch, ref = data["commit"]) - journal.send(f"Branch {branch} created in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Branch {branch} created in repo {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "created"}), 201 @app.route("/ogrepository/v1/git/repositories//branches/", methods=['DELETE']) @@ -2465,7 +2465,7 @@ def git_delete_branch(repo, branch): repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if not os.path.isdir(repo_path): - journal.send(f"Can't delete branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't delete branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return {"error": "Repository not found"}, 404 git_repo = git.Repo(repo_path) @@ -2473,11 +2473,11 @@ def git_delete_branch(repo, branch): if not branch in git_repo.branches: - journal.send(f"Can't delete branch. Not found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't delete branch. Not found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Branch not found"}), 404 git_repo.delete_head(branch) - journal.send(f"Branch {branch} deleted in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Branch {branch} deleted in repo {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "deleted"}), 200 @@ -2497,7 +2497,7 @@ def git_list_tags(repo): """ repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if not os.path.isdir(repo_path): - journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 git_repo = git.Repo(repo_path) @@ -2530,7 +2530,7 @@ def git_create_tag(repo, tag): repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if not os.path.isdir(repo_path): - journal.send(f"Can't create tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 git_repo = git.Repo(repo_path) @@ -2538,20 +2538,20 @@ def git_create_tag(repo, tag): data = request.json if data is None: - journal.send(f"Can't create tag. JSON post data missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create tag. JSON post data missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "Parameters missing"}), 400 if not "commit" in data: - journal.send(f"Can't create tag. Commit parameter missing", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create tag. Commit parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "commit parameter missing"}), 400 if tag in git_repo.tags: - journal.send(f"Can't create tag. Already found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't create tag. Already found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Tag already exists"}), 409 git_repo.create_tag(tag, ref = data["commit"]) - journal.send(f"Tag {tag} created in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Tag {tag} created in repo {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "created"}), 200 @app.route("/ogrepository/v1/git/repositories//tags/", methods=['DELETE']) @@ -2569,7 +2569,7 @@ def git_delete_tag(repo, tag): repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if not os.path.isdir(repo_path): - journal.send(f"Can't delete tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't delete tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return {"error": "Repository not found"}, 404 git_repo = git.Repo(repo_path) @@ -2577,11 +2577,11 @@ def git_delete_tag(repo, tag): if not tag in git_repo.tags: - journal.send(f"Can't delete tag. Not found in repository {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Can't delete tag. Not found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Tag not found"}), 404 git_repo.delete_head(tag) - journal.send(f"Tag {tag} deleted in repo {repo}", PRIORITY=journal.LOG_ERROR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + journal.send(f"Tag {tag} deleted in repo {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "deleted"}), 200 From 9f87997722b9e94cf16a99257be77346cbb405d3 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 7 May 2025 09:52:06 +0200 Subject: [PATCH 06/26] Add git repo synchronization --- api/repo_api.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/api/repo_api.py b/api/repo_api.py index 0df2ff9..00a5c04 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2235,6 +2235,34 @@ def git_compact_repository_task(repo, job_id): journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") recall_ogcore(data) +def git_sync_repository_task(repo, remote_repository, job_id): + journal.send("Running function 'git_sync_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + git_repo = git.Repo(git_repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', git_repo_path) + + # Recreate the remote every time, it might change + if "backup" in git_repo.remotes: + git_repo.delete_remote("backup") + + backup_repo = git_repo.create_remote("backup", remote_repository) + pushed_references = backup_repo.push("*:*") + results = [] + + # This gets returned to the API + for ref in pushed_references: + results = results + [ {"local_ref" : ref.local_ref.name, "remote_ref" : ref.remote_ref.name, "summary" : ref.summary }] + + data = { + 'job_id': job_id, + 'success': True, + 'updated_references' : results + } + + journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + recall_ogcore(data) + @app.route("/ogrepository/v1/git/repositories", methods=['GET']) def git_list_repositories(): @@ -2354,7 +2382,34 @@ def git_delete_repository(): @app.route("/ogrepository/v1/git/repositories//sync", methods=['POST']) def git_sync_repository(repo): - return jsonify({"error" : "Not implemented"}), 500 + journal.send("Running endpoint 'Sincronizar repositorio Git'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + if not os.path.isdir(REPOSITORIES_BASE_PATH): + journal.send(f"Can't sync repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 + + data = request.json + + if data is None: + journal.send(f"Can't sync repository, JSON post data missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if "remote_repository" in data: + remote_repository = data["remote_repository"] + else: + journal.send(f"Can't sync repository, JSON remote_repository parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + + job_id = f"GitSync_{''.join(random.choice('0123456789abcdef') for char in range(8))}" + + threading.Thread(target=git_sync_repository_task, args=(repo, remote_repository, job_id,)).start() + + return jsonify({ + "success": True, + "output": "Synchronizing...", + "job_id": job_id + }), 200 @app.route("/ogrepository/v1/git/repositories//backup", methods=['POST']) def git_backup_repository(repo): From c949cfb9a94e7fc003435d7bca6432cd16b6e465 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 7 May 2025 16:27:05 +0200 Subject: [PATCH 07/26] Backup endpoint --- api/repo_api.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/api/repo_api.py b/api/repo_api.py index 00a5c04..517bc11 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -60,6 +60,7 @@ import sys import git import pkgutil import importlib +import paramiko # -------------------------------------------------------------------------------------------- @@ -2263,6 +2264,31 @@ def git_sync_repository_task(repo, remote_repository, job_id): journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") recall_ogcore(data) +def git_backup_repository_task(repo, params, job_id): + journal.send("Running function 'git_sync_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + git_repo = git.Repo(git_repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', git_repo_path) + + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + ssh.connect(params["ssh_server"], params["ssh_port"], params["ssh_user"]) + sftp = ssh.open_sftp() + + with sftp.file(params["filename"], mode='wb+') as remote_file: + git_repo.archive(remote_file, format="tar.gz") + + data = { + 'job_id': job_id, + 'success': True, + } + + journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + recall_ogcore(data) + + @app.route("/ogrepository/v1/git/repositories", methods=['GET']) def git_list_repositories(): @@ -2413,7 +2439,45 @@ def git_sync_repository(repo): @app.route("/ogrepository/v1/git/repositories//backup", methods=['POST']) def git_backup_repository(repo): - return jsonify({"error" : "Not implemented"}), 500 + journal.send("Running endpoint 'Backup de repositorio Git'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + if not os.path.isdir(REPOSITORIES_BASE_PATH): + journal.send(f"Can't backup repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 + + data = request.json + + if data is None: + journal.send(f"Can't backup repository, JSON post data missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "ssh_server" in data: + journal.send(f"Can't sync repository, JSON ssh_server parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "ssh_port" in data: + journal.send(f"Can't sync repository, JSON ssh_port parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "ssh_user" in data: + journal.send(f"Can't sync repository, JSON ssh_user parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "filename" in data: + journal.send(f"Can't sync repository, JSON filename parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + + job_id = f"GitBackup_{''.join(random.choice('0123456789abcdef') for char in range(8))}" + + threading.Thread(target=git_backup_repository_task, args=(repo, data, job_id,)).start() + + return jsonify({ + "success": True, + "output": "Backing up...", + "job_id": job_id + }), 200 + @app.route("/ogrepository/v1/git/repositories//compact", methods=['POST']) def git_compact_repository(repo): From 28f2537aed850fa9bd52d18df74336b0077d0ebc Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Fri, 9 May 2025 12:51:39 +0200 Subject: [PATCH 08/26] Swagger fix: use the right name in example --- api/swagger.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index 1508c93..9e7d057 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2098,7 +2098,7 @@ paths: schema: type: object properties: - name: + commit: type: string example: HEAD responses: @@ -2254,7 +2254,7 @@ paths: schema: type: object properties: - name: + commit: type: string example: HEAD responses: From 984b251615034492400ce08245e9edb149136529 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Fri, 9 May 2025 13:33:00 +0200 Subject: [PATCH 09/26] Use annotated tags --- api/repo_api.py | 21 +++++++++++++++++++-- api/swagger.yaml | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 517bc11..a96983d 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2625,7 +2625,20 @@ def git_list_tags(repo): tags = [] for tag in git_repo.tags: - tags = tags + [tag.name] + tag_info = { + "name" : tag.name, + "commit" : tag.commit.hexsha, + "committer" : tag.commit.committer.name, + "committed_datetime" : tag.commit.committed_datetime.timestamp() + } + + if not tag.tag is None: + tag_info["message"] = tag.tag.message + tag_info["tagged_date"] = tag.tag.tagged_date + tag_info["tagger"] = tag.tag.tagger.name + tag_info["tagger_tz_offset"] = tag.tag.tagger_tz_offset + + tags = tags + [ tag_info ] journal.send(f"Returning {len(tags)} branches", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return { @@ -2664,11 +2677,15 @@ def git_create_tag(repo, tag): journal.send(f"Can't create tag. Commit parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "commit parameter missing"}), 400 + commit_message = "" + if "message" in data: + commit_message = data["message"] + if tag in git_repo.tags: journal.send(f"Can't create tag. Already found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Tag already exists"}), 409 - git_repo.create_tag(tag, ref = data["commit"]) + git_repo.create_tag(tag, ref = data["commit"], message = commit_message) journal.send(f"Tag {tag} created in repo {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "created"}), 200 diff --git a/api/swagger.yaml b/api/swagger.yaml index 9e7d057..f79b0e6 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2058,8 +2058,36 @@ paths: tags: type: array items: - type: string - example: v0.1 + type: object + properties: + name: + type: string + example: v0.1 + commit: + type: string + example: db8e84d5d2548f589ee503c1c6d5003cc6a0d803 + committer: + type: string + example: John Smith + committed_datetime: + type: int + example: 1745360193 + message: + type: string + example: "Initial release" + required: False + tagged_date: + type: int + example: 1745360194 + required: False + tagger: + type: string + example: John Smith + required: False + tagger_tz_offset: + type: int + example: -7200 + required: False "500": description: "Excepción" @@ -2095,12 +2123,18 @@ paths: required: true description: | * **commit** - Commit al que apunta el tag nuevo. Puede ser un nombre de otra rama/tag. + * **message** - Mensaje descriptivo para el tag. Opcional, si no se especifica se asume una cadena vacía. schema: type: object properties: commit: type: string example: HEAD + required: True + message: + type: string + example: Version 1.0 + required: False responses: "201": description: "Tag creado" From bff65cde01ac8752375760471afcf23dde726060 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 12 May 2025 09:15:16 +0200 Subject: [PATCH 10/26] Fix repository already exists error code --- api/repo_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index a96983d..fad305a 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2376,7 +2376,7 @@ def git_create_repository(): Returns: Response: A JSON response with a status message and HTTP status code. - - 200: If the repository already exists. + - 409: If the repository already exists. - 201: If the repository is successfully created. """ data = request.json @@ -2390,7 +2390,7 @@ def git_create_repository(): repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") if os.path.isdir(repo_path): journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - return jsonify({"status": "Repository already exists"}), 200 + return jsonify({"status": "Repository already exists"}), 409 module = _load_installer() print(f"Got {module}") @@ -2540,7 +2540,7 @@ def git_create_branch(repo, branch): Response: A JSON response containing a creation status - 201: A JSON object with a "status" key containing "created" - 404: A JSON object with an "error" key containing the message "Repository not found" - - 409: A JSON object with an "error" key containing the message "Tag already exists" + - 409: A JSON object with an "error" key containing the message "Branch already exists" """ repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") From b04bf3c5b1a6ac63cc1278115f5c6a99c058d378 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 12 May 2025 11:50:26 +0200 Subject: [PATCH 11/26] Cambiar endpoint de crear tags --- api/repo_api.py | 20 ++++++++++++++++---- api/swagger.yaml | 23 ++++++++++------------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index fad305a..19fd176 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2529,8 +2529,8 @@ def git_get_branches(repo): "branches": branches } -@app.route("/ogrepository/v1/git/repositories//branches/", methods=['POST']) -def git_create_branch(repo, branch): +@app.route("/ogrepository/v1/git/repositories//branches", methods=['POST']) +def git_create_branch(repo): """Create a given branch in a given repository Args: @@ -2560,6 +2560,12 @@ def git_create_branch(repo, branch): journal.send(f"Can't create branch. Commit parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "commit parameter missing"}), 400 + if not "name" in data: + journal.send(f"Can't create branch. Name parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "name parameter missing"}), 400 + + branch = data["name"] + if branch in git_repo.branches: journal.send(f"Can't create branch. Already found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Branch already exists"}), 409 @@ -2646,8 +2652,8 @@ def git_list_tags(repo): } -@app.route("/ogrepository/v1/git/repositories//tags/", methods=['POST']) -def git_create_tag(repo, tag): +@app.route("/ogrepository/v1/git/repositories//tags", methods=['POST']) +def git_create_tag(repo): """Create a given tag in a given repository Args: @@ -2677,7 +2683,13 @@ def git_create_tag(repo, tag): journal.send(f"Can't create tag. Commit parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "commit parameter missing"}), 400 + if not "name" in data: + journal.send(f"Can't create tag. Name parameter missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "name parameter missing"}), 400 + commit_message = "" + tag = data["name"] + if "message" in data: commit_message = data["message"] diff --git a/api/swagger.yaml b/api/swagger.yaml index f79b0e6..a619772 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2100,7 +2100,6 @@ paths: exception: type: string example: "(Exception description)" - /ogrepository/v1/git/repositories/{repository}/tags/{tag}: post: summary: "Crear tag" description: | @@ -2113,20 +2112,20 @@ paths: required: true type: string description: "Nombre de repositorio" - - name: tag - in: path - required: true - type: string - description: "Tag del repositorio" - name: JSON in: body required: true description: | + * **name** - Nombre del tag * **commit** - Commit al que apunta el tag nuevo. Puede ser un nombre de otra rama/tag. * **message** - Mensaje descriptivo para el tag. Opcional, si no se especifica se asume una cadena vacía. schema: type: object properties: + name: + type: string + example: v1.0 + required: True commit: type: string example: HEAD @@ -2261,8 +2260,6 @@ paths: error: type: string example: "Branch already exists" - - /ogrepository/v1/git/repositories/{repository}/branches/{branch}: post: summary: "Crear branch" description: | @@ -2275,11 +2272,6 @@ paths: required: true type: string description: "Nombre de repositorio" - - name: branch - in: path - required: true - type: string - description: "Branch del repositorio" - name: JSON in: body required: true @@ -2288,9 +2280,14 @@ paths: schema: type: object properties: + name: + type: string + example: devel + required: True commit: type: string example: HEAD + required: True responses: "201": description: "Rama creada" From 3c2fc27e2e1ae3de8ae2acb3bfbe4a3d5552c75e Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 4 Jun 2025 16:48:08 +0200 Subject: [PATCH 12/26] Package build fixes --- debian/changelog | 5 +++++ debian/ogrepository.postinst | 24 ++++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index a7337d9..3ac13fe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,8 @@ +ogrepository (1.1.0) unstable; urgency=medium + * OgGit + + -- Tu Nombre Tue, 11 Mar 2025 04:43:58 +0000 + ogrepository (1.0.0+deb-packages20250311-1) unstable; urgency=medium * First debian version diff --git a/debian/ogrepository.postinst b/debian/ogrepository.postinst index a356423..fff9a6a 100755 --- a/debian/ogrepository.postinst +++ b/debian/ogrepository.postinst @@ -1,7 +1,7 @@ #!/bin/sh set -e -set -x +set -x . /usr/share/debconf/confmodule @@ -38,7 +38,7 @@ SAMBA_PASS="$RET" USER="opengnsys" -# Provisionar base de datos si es necesario en caso de instalación. +# Provisionar base de datos si es necesario en caso de instalación. # Detectar si es una instalación nueva o una actualización @@ -53,6 +53,16 @@ USER="opengnsys" # done # " if [ "$1" = "configure" ] && [ -z "$2" ]; then + systemd-run --no-block /bin/bash -c " +sleep 10; +apt update -y; +for pkg in bittorrent bittornado ctorrent; do + if ! dpkg -l | grep -qw \"\$pkg\"; then + apt install -y \"\$pkg\" + fi +done +" + sed -i "s/%%OGREPOSITORY_USER%%/$SAMBA_USER/g" /etc/systemd/system/ogrepo-api.service sed -i "s/%%OGREPOSITORY_USER%%/$SAMBA_USER/g" /etc/samba/ogrepo-smb.conf @@ -68,7 +78,7 @@ if [ "$1" = "configure" ] && [ -z "$2" ]; then chmod 700 $OPENGNSYS_HOME/.ssh chmod 600 $OPENGNSYS_HOME/.ssh/id_ed25519 chmod 644 $OPENGNSYS_HOME/.ssh/id_ed25519.pub - # Genera authorized_keys + # Genera authorized_keys cat $OPENGNSYS_HOME/.ssh/id_ed25519.pub >> $OPENGNSYS_HOME/.ssh/authorized_keys chmod 600 $OPENGNSYS_HOME/.ssh/authorized_keys chown -R opengnsys:opengnsys $OPENGNSYS_HOME/.ssh @@ -89,11 +99,11 @@ if [ "$1" = "configure" ] && [ -z "$2" ]; then sed -i "s/OGCOREIP/$OGCORE_IP/g" /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg # Copiar sudoers file ogrepository/etc/opengnsys-repository - cp /opt/opengnsys/ogrepository/etc/opengnsys-repository /etc/sudoers.d/opengnsys + cp /opt/opengnsys/ogrepository/etc/opengnsys-repository /etc/sudoers.d/opengnsys elif [ "$1" = "configure" ] && [ -n "$2" ]; then echo "Actualización desde la versión $2" - # Recopy static files without configuration - cp /opt/opengnsys/ogrepository/etc/opengnsys-repository /etc/sudoers.d/opengnsys-repository + # Recopy static files without configuration + cp /opt/opengnsys/ogrepository/etc/opengnsys-repository /etc/sudoers.d/opengnsys-repository OPENGNSYS_HOME=$(getent passwd opengnsys | cut -d: -f6) # Create .ssh directory mkdir -p $OPENGNSYS_HOME/.ssh @@ -115,8 +125,10 @@ fi # Cambiar la propiedad de los archivos al usuario especificado chown opengnsys:www-data /opt/opengnsys/ chown -R opengnsys:www-data /opt/opengnsys/ogrepository +chmod 755 /opt/opengnsys chmod 755 /opt/opengnsys/ogrepository/bin/* + # Install http server stuff # Reiniciar servicios si es necesario # systemctl restart nombre_del_servicio From da5d6fd8c54fd3f2aeea2059118c452492ffd483 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Thu, 5 Jun 2025 09:47:59 +0200 Subject: [PATCH 13/26] Fix base path handling --- api/repo_api.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 19fd176..1cb115e 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -54,7 +54,8 @@ config_file = '/opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg' # GIT # -------------------------------------------------------------------------------------------- -REPOSITORIES_BASE_PATH = "/opt/opengnsys/ogrepository/oggit/git/oggit/" +REPOSITORIES_BASE_PATH = "/opt/opengnsys/ogrepository/oggit/git/" +OGGIT_USER = "oggit" import sys import git @@ -2222,7 +2223,7 @@ def rename_image(): def git_compact_repository_task(repo, job_id): journal.send("Running function 'git_compact_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") git_repo = git.Repo(git_repo_path) git_repo.git.config('--global', '--add', 'safe.directory', git_repo_path) @@ -2239,7 +2240,7 @@ def git_compact_repository_task(repo, job_id): def git_sync_repository_task(repo, remote_repository, job_id): journal.send("Running function 'git_sync_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") git_repo = git.Repo(git_repo_path) git_repo.git.config('--global', '--add', 'safe.directory', git_repo_path) @@ -2267,7 +2268,7 @@ def git_sync_repository_task(repo, remote_repository, job_id): def git_backup_repository_task(repo, params, job_id): journal.send("Running function 'git_sync_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + git_repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") git_repo = git.Repo(git_repo_path) git_repo.git.config('--global', '--add', 'safe.directory', git_repo_path) @@ -2317,7 +2318,7 @@ def git_list_repositories(): return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 repos = [] - for entry in os.scandir(REPOSITORIES_BASE_PATH): + for entry in os.scandir(os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER)): if entry.is_dir(follow_symlinks=False) and os.path.isfile(os.path.join(entry.path, "HEAD")): name = entry.name if name.endswith(".git"): @@ -2387,7 +2388,7 @@ def git_create_repository(): repo = data["name"] - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if os.path.isdir(repo_path): journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "Repository already exists"}), 409 @@ -2511,7 +2512,7 @@ def git_get_branches(repo): - 200: A JSON object with a "branches" key containing a list of branch names. - 404: A JSON object with an "error" key containing the message "Repository not found" if the repository does not exist. """ - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if not os.path.isdir(repo_path): journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 @@ -2543,7 +2544,7 @@ def git_create_branch(repo): - 409: A JSON object with an "error" key containing the message "Branch already exists" """ - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if not os.path.isdir(repo_path): journal.send(f"Can't create branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 @@ -2588,7 +2589,7 @@ def git_delete_branch(repo, branch): - 404: A JSON object with an "error" key containing the message "Repository not found" or "Branch not found" """ - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if not os.path.isdir(repo_path): journal.send(f"Can't delete branch. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return {"error": "Repository not found"}, 404 @@ -2620,7 +2621,7 @@ def git_list_tags(repo): - 200: A JSON object with a "branches" key containing a list of tag names. - 404: A JSON object with an "error" key containing the message "Repository not found" if the repository does not exist. """ - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if not os.path.isdir(repo_path): journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 @@ -2666,7 +2667,7 @@ def git_create_tag(repo): - 409: A JSON object with an "error" key containing the message "Tag already exists" """ - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if not os.path.isdir(repo_path): journal.send(f"Can't create tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Repository not found"}), 404 @@ -2715,7 +2716,7 @@ def git_delete_tag(repo, tag): - 404: A JSON object with an "error" key containing the message "Repository not found" or "Tag not found" """ - repo_path = os.path.join(REPOSITORIES_BASE_PATH, repo + ".git") + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") if not os.path.isdir(repo_path): journal.send(f"Can't delete tag. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return {"error": "Repository not found"}, 404 From 05b963d61125ac22bc44f7fc96f7495cacdec447 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Fri, 6 Jun 2025 08:54:45 +0200 Subject: [PATCH 14/26] Fix path handling when there are no repositories --- api/repo_api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 1cb115e..9d21f2b 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2318,13 +2318,17 @@ def git_list_repositories(): return jsonify({"error": "Repository storage not found, git functionality may not be installed."}), 500 repos = [] - for entry in os.scandir(os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER)): - if entry.is_dir(follow_symlinks=False) and os.path.isfile(os.path.join(entry.path, "HEAD")): - name = entry.name - if name.endswith(".git"): - name = name[:-4] + if os.path.exists(os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER)): + # If the base path exists, but the OGGIT_USER subpath doesn't, it means we've got an empty + # install. OgGit is present, but there's no repos yet. - repos = repos + [name] + for entry in os.scandir(os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER)): + if entry.is_dir(follow_symlinks=False) and os.path.isfile(os.path.join(entry.path, "HEAD")): + name = entry.name + if name.endswith(".git"): + name = name[:-4] + + repos = repos + [name] journal.send(f"Returning {len(repos)} repositories", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") From 8dcd1faefd517a37a9df3423869714ecf029bceb Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Fri, 6 Jun 2025 10:11:29 +0200 Subject: [PATCH 15/26] Add another search path for installer module --- api/repo_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/repo_api.py b/api/repo_api.py index 9d21f2b..87ce3c4 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2354,13 +2354,18 @@ def _load_installer(): script_dir = os.path.dirname(os.path.realpath(__file__)) - system_lib_path = '/opt/opengnsys/oggit/bin/' + system_bin_path = '/opt/opengnsys/oggit/bin/' + system_lib_path = '/opt/opengnsys/oggit/lib/' devel_lib_path = os.path.join(script_dir, "../../oggit/installer") if devel_lib_path not in sys.path and os.path.isdir(devel_lib_path): journal.send(f"Using {devel_lib_path} development library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") sys.path.append(devel_lib_path) + if system_bin_path not in sys.path: + journal.send(f"Using {system_bin_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") + sys.path.append(system_bin_path) + if system_lib_path not in sys.path: journal.send(f"Using {system_lib_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") sys.path.append(system_lib_path) From e3555627146ba3aca0bec9ac8fa1aa561a113398 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Fri, 6 Jun 2025 10:15:45 +0200 Subject: [PATCH 16/26] Fix path --- api/repo_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 87ce3c4..e07d091 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2354,8 +2354,8 @@ def _load_installer(): script_dir = os.path.dirname(os.path.realpath(__file__)) - system_bin_path = '/opt/opengnsys/oggit/bin/' - system_lib_path = '/opt/opengnsys/oggit/lib/' + system_bin_path = '/opt/opengnsys/ogrepository/oggit/bin/' + system_lib_path = '/opt/opengnsys/ogrepository/oggit/lib/' devel_lib_path = os.path.join(script_dir, "../../oggit/installer") if devel_lib_path not in sys.path and os.path.isdir(devel_lib_path): From a1f0561f5c69dd400ad900e9548e96c5f6968deb Mon Sep 17 00:00:00 2001 From: Vadim Trochinsky Date: Mon, 16 Jun 2025 15:55:40 +0200 Subject: [PATCH 17/26] Add SSH key endpoint --- api/repo_api.py | 101 +++++++++++++++++++++++++++++++++++++++++++++++ api/swagger.yaml | 58 +++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) diff --git a/api/repo_api.py b/api/repo_api.py index e07d091..f7b2d86 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2290,6 +2290,38 @@ def git_backup_repository_task(repo, params, job_id): recall_ogcore(data) +def git_add_sshkey_task(oglive, description, job_id): + module = _load_installer() + print(f"Got {module}") + OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller') + installer = OpengnsysGitInstaller() + + results = installer.add_ssh_key_from_squashfs(oglive_file = oglive) + keys_added = 0 + keys_existed = 0 + keys_failed = 0 + + for status, message in results: + if status == 200 or status == 201: + keys_added = keys_added + 1 + elif status == 422: + keys_existed = keys_existed + 1 + else: + keys_failed = keys_failed + 1 + journal.send(f"Unrecognized reply from forgejo: code {status}, content {message}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + + data = { + 'job_id': job_id, + 'keys_added' : keys_added, + 'keys_failed' : keys_failed, + 'keys_existed' : keys_existed, + 'output' : message + } + + journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + recall_ogcore(data) + @app.route("/ogrepository/v1/git/repositories", methods=['GET']) def git_list_repositories(): @@ -2746,6 +2778,75 @@ def git_delete_tag(repo, tag): +@app.route("/ogrepository/v1/git/ssh_key", methods=['POST']) +def git_add_sshkey(): + """Add a SSH key + + Args: + ssh_key (str): The SSH key + oglive (str): URL to an oglive image from which to extract the key. May be a local file or HTTP. + description (str): Description for the SSH key + + Returns: + Response: A JSON response containing a list of tag names or an error message if the key can't be added. + - 200: A JSON object with a "status" key containing "added" + + """ + + data = request.json + + if data is None: + journal.send(f"Can't add SSH keys, POST data missing", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing"}), 400 + + if not "description" in data: + data["description"] = "" + + if not ("ssh_key" in data or "oglive" in data): + journal.send(f"Can't add SSH keys, either ssh_key or oglive is required", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Parameters missing, specify ssh_key or oglive"}), 400 + + + + module = _load_installer() + print(f"Got {module}") + OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller') + installer = OpengnsysGitInstaller() + + if "oglive" in data: + job_id = f"GitSshKey_{''.join(random.choice('0123456789abcdef') for char in range(8))}" + threading.Thread(target=git_add_sshkey_task, args=(data["oglive"], data["description"], job_id)).start() + return jsonify({ + "success": True, + "output": "Extracting key from ogLive...", + "job_id": job_id + }), 200 + else: + status, content = installer.add_forgejo_sshkey(data["ssh_key"], data["description"]) + message = "Result unrecognized" + success = False + httpcode = 500 + + if status == 200 or status == 201: + message = "SSH key added" + success = True + httpcode = 200 + elif status == 422: + message = "SSH key already existed" + success = True + httpcode = 200 + else: + message = "Unrecognized reply from forgejo" + success = False + httpcode = status + journal.send(f"Unrecognized reply from forgejo: code {status}, content {content}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + return jsonify({ + "success": success, + "output": message, + }), httpcode + + # -------------------------------------------------------------------------------------------- diff --git a/api/swagger.yaml b/api/swagger.yaml index a619772..18e1ba9 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1971,6 +1971,64 @@ paths: # # ----------------------------------------------------------- + /ogrepository/v1/git/ssh_key: + post: + summary: "Agregar clave de SSH de ogLive" + description: | + Agrega una clave de SSH que va a usarse desde el ogLive + para interactuar con Git. + + Es necesario especificar **ssh_key** o **oglive**. + + Especificando **ssh_key** se agrega la cclave especificada directamente. + + Especificando **oglive** se descarga si es necesario el .iso, se monta, y se extrae la clave. + Esta acción se hace en el fondo, y se devuelve un job_id. + + tags: + - "Git" + parameters: + - name: JSON + in: body + required: true + description: | + * **ssh_key** - Clave de SSH (opcional) + * **oglive** - URL a ogLive (opcional, NO USAR DE MOMENTO) + * **description** - Descripcion (opcional) + schema: + type: object + properties: + ssh_key: + type: string + example: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINEOttwhJ+9THRZ1Zv/6QUwPUDq1X7opG9V7EFLVWxQV" + required: False + description: + type: string + example: "OgLive r20250518" + required: False + oglive: + type: string + example: "https://ognproject.evlt.uma.es/oglive/ogLive-noble-6.8.0-31-generic-amd64-r20250518.cf13a6d_20250519.iso" + required: False + responses: + "200": + description: "Exito" + schema: + type: object + properties: + success: + type: boolean + example: True + required: True + output: + type: string + example: "SSH key added" + required: True + job_id: + type: string + example: "GitSshKey_873f353f" + required: False + /ogrepository/v1/git/repositories: get: summary: "Obtener lista de repositorios" From 87286829369d9d202a487621906cb9cd772ddc74 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 16 Jun 2025 23:42:10 +0200 Subject: [PATCH 18/26] update dependencies --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 1ee440d..d6ea418 100644 --- a/debian/control +++ b/debian/control @@ -8,7 +8,7 @@ Build-Depends: debhelper-compat (= 12) Package: ogrepository Architecture: all Pre-Depends: debian-archive-keyring , debconf (>= 1.5.0), -Depends: ${misc:Depends}, git, python3, python3-pip, python3-flask, python3-paramiko, python3-psutil, python3-flasgger, samba, gunicorn, wakeonlan , lzop , partclone , qemu-utils , udpcast, uftp, mktorrent, aria2 , opengnsys-opentracker +Depends: ${misc:Depends}, git, python3, python3-pip, python3-flask, python3-paramiko, python3-psutil, python3-flasgger, samba, gunicorn, wakeonlan , lzop , partclone , qemu-utils , udpcast, uftp, mktorrent, aria2 , opengnsys-opentracker, opengnsys-gitinstaller, opengnsys-forgejo Description: Ogrepsoitory Package This package provides Ogrepository service. From 3992ff27f0ed9fa2fa64713fa389be61434a7151 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Thu, 5 Jun 2025 09:56:22 +0200 Subject: [PATCH 19/26] update changelog --- debian/changelog | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/debian/changelog b/debian/changelog index 3ac13fe..369f622 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +ogrepository (1.1.2) UNRELEASED; urgency=medium + + * Fix path handling when there are no repositories + * Add another search path for installer module + * Fix path handling + * Add SSH key endpoint + * Update dependencies + + -- OpenGnsys Tue, 30 Jun 2025 15:20:23 +0000 + +ogrepository (1.1.1) unstable; urgency=medium + * OgGit - fix repo path handling + + -- Tu Nombre Jue, 5 Jun 2025 08:43:58 +0000 + ogrepository (1.1.0) unstable; urgency=medium * OgGit From 228bd85aa83c4065698148e14cd87f980f802a0d Mon Sep 17 00:00:00 2001 From: Vadim Trochinsky Date: Mon, 23 Jun 2025 15:30:25 +0200 Subject: [PATCH 20/26] Add commit listing --- api/repo_api.py | 68 ++++++++++++++++++++++++++++++++++++ api/swagger.yaml | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) diff --git a/api/repo_api.py b/api/repo_api.py index f7b2d86..9d68e42 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2571,6 +2571,74 @@ def git_get_branches(repo): "branches": branches } +@app.route("/ogrepository/v1/git/repositories//branches//commits", methods=['GET']) +def git_get_commits(repo, branch): + """ + Retrieve the list of commits in a branch in a given repository. + + Args: + repo (str): The name of the repository. + + Returns: + Response: A JSON response containing a list of commits + - 200: A JSON object with a "branches" key containing a list of branch names. + - 404: A JSON object with an "error" key containing the message "Repository not found" if the repository does not exist. + """ + repo_path = os.path.join(REPOSITORIES_BASE_PATH, OGGIT_USER, repo + ".git") + if not os.path.isdir(repo_path): + journal.send(f"Can't list repositories. Repository storage at {REPOSITORIES_BASE_PATH} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Repository not found"}), 404 + + + max_commits = request.args.get('max_commits') or 100 + skip_commits = request.args.get('skip') or 0 + + + git_repo = git.Repo(repo_path) + git_repo.git.config('--global', '--add', 'safe.directory', repo_path) + + + if not branch in git_repo.branches: + journal.send(f"Branch {branch} not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error": "Branch not found"}), 404 + + + # Lookup table for hash/tag + hash_to_tag = {} + for tag in git_repo.tags: + sha = tag.commit.hexsha + if not sha in hash_to_tag: + hash_to_tag[sha] = [] + + hash_to_tag[sha] = hash_to_tag[sha] + [ tag.name ] + + + commits = [] + for com in git_repo.iter_commits(branch, max_count = max_commits, skip = skip_commits): + tag_list = [] + + if com.hexsha in hash_to_tag: + tag_list = hash_to_tag[com.hexsha] + + + commits = commits + [ + { + "hexsha" : com.hexsha, + "message" : com.message, + "committed_date" : com.committed_date, + "tags" : tag_list, + "size" : com.size, + "stats_total" : com.stats.total + } + ] + + + journal.send(f"Returning {len(commits)} commits", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return { + "commits": commits + } + + @app.route("/ogrepository/v1/git/repositories//branches", methods=['POST']) def git_create_branch(repo): """Create a given branch in a given repository diff --git a/api/swagger.yaml b/api/swagger.yaml index 18e1ba9..15dd8c7 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2266,6 +2266,96 @@ paths: exception: type: string example: "(Exception description)" + /ogrepository/v1/git/repositories/{repository}/branches/{branch}/commits: + get: + summary: "Obtener lista de commits en una rama" + description: | + Devuelve una lista de commits de Git + tags: + - "Git" + parameters: + - name: repository + in: path + required: true + type: string + description: "Nombre de repositorio" + - name: branch + in: path + required: true + type: string + description: "Rama dentro del repositorio" + - name: max_commits + in: query + required: false + type: int + description: "Máximo de commits a obtener" + - name: skip + in: query + required: false + type: int + description: "Commits a saltar (para paginación)" + responses: + "200": + description: "Lista de commits" + schema: + type: object + properties: + commits: + type: array + items: + type: object + properties: + committed_date: + type: int + example: 1745360193 + hexsha: + type: string + example: "db8e84d5d2548f589ee503c1c6d5003cc6a0d803" + message: + type: string + example: "Install updates" + size: + type: int + example: 67108864 + tags: + type: array + example: ["updates"] + stats_total: + type: object + properties: + deletion: + type: int + example: 0 + files: + type: int + example: 1 + insertions: + type: int + example: 1 + lines: + type: int + example: 100 + + "500": + description: "Excepción" + schema: + type: object + properties: + success: + type: boolean + example: false + exception: + type: string + example: "(Exception description)" + "404": + description: "El repositorio o branch no existe" + schema: + type: object + properties: + error: + type: string + example: "Repository not found" + /ogrepository/v1/git/repositories/{repository}/branches: get: summary: "Obtener lista de branches" From 05b51de851bea8352c696f4a25def531eceac12a Mon Sep 17 00:00:00 2001 From: Vadim Trochinsky Date: Tue, 24 Jun 2025 16:35:29 +0200 Subject: [PATCH 21/26] Better error handling for bad tag names --- api/repo_api.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/repo_api.py b/api/repo_api.py index 9d68e42..9950ec8 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2807,7 +2807,17 @@ def git_create_tag(repo): journal.send(f"Can't create tag. Already found in repository {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error": "Tag already exists"}), 409 - git_repo.create_tag(tag, ref = data["commit"], message = commit_message) + try: + git_repo.create_tag(tag, ref = data["commit"], message = commit_message) + except git.exc.GitCommandError as ge: + if "not a valid tag name" in ge.stderr: + journal.send(f"Tag name {tag} is invalid", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Invalid tag name"}), 400 + else: + journal.send(f"Git error {ge}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + return jsonify({"error" : "Error when performing git command"}), 500 + + journal.send(f"Tag {tag} created in repo {repo}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "created"}), 200 From 199ef8c5e1621cae1ce3cc3eb32b11917347876b Mon Sep 17 00:00:00 2001 From: Vadim Trochinsky Date: Wed, 25 Jun 2025 15:17:45 +0200 Subject: [PATCH 22/26] ref #2317 Fix performance problem, remove stats --- api/repo_api.py | 3 +-- api/swagger.yaml | 15 --------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 9950ec8..571f506 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2627,8 +2627,7 @@ def git_get_commits(repo, branch): "message" : com.message, "committed_date" : com.committed_date, "tags" : tag_list, - "size" : com.size, - "stats_total" : com.stats.total + "size" : com.size } ] diff --git a/api/swagger.yaml b/api/swagger.yaml index 15dd8c7..975bb29 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2320,21 +2320,6 @@ paths: tags: type: array example: ["updates"] - stats_total: - type: object - properties: - deletion: - type: int - example: 0 - files: - type: int - example: 1 - insertions: - type: int - example: 1 - lines: - type: int - example: 100 "500": description: "Excepción" From 83b0fd7c03f38a401851c9b76ab2db47948d188c Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Tue, 1 Jul 2025 15:30:59 +0200 Subject: [PATCH 23/26] refs #2363 Add python3-git dependency --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index d6ea418..9c16548 100644 --- a/debian/control +++ b/debian/control @@ -8,7 +8,7 @@ Build-Depends: debhelper-compat (= 12) Package: ogrepository Architecture: all Pre-Depends: debian-archive-keyring , debconf (>= 1.5.0), -Depends: ${misc:Depends}, git, python3, python3-pip, python3-flask, python3-paramiko, python3-psutil, python3-flasgger, samba, gunicorn, wakeonlan , lzop , partclone , qemu-utils , udpcast, uftp, mktorrent, aria2 , opengnsys-opentracker, opengnsys-gitinstaller, opengnsys-forgejo +Depends: ${misc:Depends}, git, python3, python3-pip, python3-flask, python3-paramiko, python3-psutil, python3-flasgger, samba, gunicorn, wakeonlan , lzop , partclone , qemu-utils , udpcast, uftp, mktorrent, aria2 , opengnsys-opentracker, opengnsys-forgejo, python3-git Description: Ogrepsoitory Package This package provides Ogrepository service. From 037ac63c97c039ff9d071ac18761b1b288188ba5 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Tue, 1 Jul 2025 15:31:32 +0200 Subject: [PATCH 24/26] refs #2363 Reduce number of required dependencies --- api/repo_api.py | 183 +++++++++++++++++------------------------------ debian/changelog | 14 ++++ 2 files changed, 81 insertions(+), 116 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index 571f506..c52ab95 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -56,11 +56,10 @@ config_file = '/opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg' REPOSITORIES_BASE_PATH = "/opt/opengnsys/ogrepository/oggit/git/" OGGIT_USER = "oggit" +OGGIT_FORGEJO_PORT = 3100 import sys import git -import pkgutil -import importlib import paramiko @@ -1603,7 +1602,7 @@ def send_p2p(): "success": False, "error": error_message }), 500 - + # Ejecutamos los scripts "runTorrentSeeder.py", que no reciben parámetros. journal.send("Running script 'runTorrentSeeder.py'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -1614,7 +1613,7 @@ def send_p2p(): sleep(10) seeder_running = search_process('aria2c', f"{param_dict['name']}.img.torrent") # El seeder se ejecuta con "aria2c" y el nombre de la imagen como parámetro - + # Evaluamos las comprobaciones anteriores, para devolver la respuesta que corresponda: if seeder_running: journal.send("'runTorrentSeeder.py' results OK (ReturnCodes: None), and processes running", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2220,6 +2219,48 @@ def rename_image(): # ----------------------------------------------------------- +def git_forgejo_post_command(url, json_data): + token = "" + with open("/opt/opengnsys/etc/ogGitApiToken.cfg", "r", encoding='utf-8') as token_file: + token = token_file.read().strip() + + if url[0] == "/": + url = url[1:] + + url = f"http://localhost:{OGGIT_FORGEJO_PORT}/{url}" + + journal.send(f"Executing forgejo POST to {url} with data {json_data}...", PRIORITY=journal.LOG_DEBUG, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + r = requests.post( + url, + json = json_data, + headers = { 'Authorization' : f"token {token}" }, + timeout = 60 + ) + + journal.send(f"Request status was {r.status_code}, content {r.content}", PRIORITY=journal.LOG_DEBUG, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + + return r.status_code, r.content.decode('utf-8') + +def git_forgejo_create_repo(repo): + return git_forgejo_post_command("/api/v1/user/repos", + { + "auto_init" : False, + "default_branch" : "main", + "description" : "", + "name" : repo, + "private" : False + }) + + +def git_forgejo_add_sshkey(pubkey, description = ""): + return git_forgejo_post_command("/api/v1/user/keys", + { + "key" : pubkey, + "read_only" : False, + "title" : description + }) + def git_compact_repository_task(repo, job_id): journal.send("Running function 'git_compact_repository_task'...", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2289,40 +2330,6 @@ def git_backup_repository_task(repo, params, job_id): journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") recall_ogcore(data) - -def git_add_sshkey_task(oglive, description, job_id): - module = _load_installer() - print(f"Got {module}") - OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller') - installer = OpengnsysGitInstaller() - - results = installer.add_ssh_key_from_squashfs(oglive_file = oglive) - keys_added = 0 - keys_existed = 0 - keys_failed = 0 - - for status, message in results: - if status == 200 or status == 201: - keys_added = keys_added + 1 - elif status == 422: - keys_existed = keys_existed + 1 - else: - keys_failed = keys_failed + 1 - journal.send(f"Unrecognized reply from forgejo: code {status}, content {message}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - - - data = { - 'job_id': job_id, - 'keys_added' : keys_added, - 'keys_failed' : keys_failed, - 'keys_existed' : keys_existed, - 'output' : message - } - - journal.send(f"Calling function 'recall_ogcore' (JOB_ID: {job_id}, SUCCESS: True)", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - recall_ogcore(data) - - @app.route("/ogrepository/v1/git/repositories", methods=['GET']) def git_list_repositories(): """ @@ -2369,42 +2376,6 @@ def git_list_repositories(): "repositories": repos }), 200 -def _load_module(module_name): - # module = importlib.util.find_spec(module_name) - module = importlib.import_module(module_name) - - if module is not None: - journal.send(f"Module {module_name} loaded successfully. Got {module}", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") - return module - - journal.send(f"Module {module_name} failed to load, not found", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="oggit-api_DEBUG") - return False - -def _load_installer(): - - journal.send(f"Loading oggit installer module", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") - - script_dir = os.path.dirname(os.path.realpath(__file__)) - - system_bin_path = '/opt/opengnsys/ogrepository/oggit/bin/' - system_lib_path = '/opt/opengnsys/ogrepository/oggit/lib/' - devel_lib_path = os.path.join(script_dir, "../../oggit/installer") - - if devel_lib_path not in sys.path and os.path.isdir(devel_lib_path): - journal.send(f"Using {devel_lib_path} development library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") - sys.path.append(devel_lib_path) - - if system_bin_path not in sys.path: - journal.send(f"Using {system_bin_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") - sys.path.append(system_bin_path) - - if system_lib_path not in sys.path: - journal.send(f"Using {system_lib_path} system library path", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="oggit-api_DEBUG") - sys.path.append(system_lib_path) - - return _load_module("opengnsys_git_installer") - - @app.route("/ogrepository/v1/git/repositories", methods=['POST']) def git_create_repository(): """ @@ -2434,11 +2405,7 @@ def git_create_repository(): journal.send(f"Can't create repository {repo}, already exists at {repo_path}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"status": "Repository already exists"}), 409 - module = _load_installer() - print(f"Got {module}") - OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller') - installer = OpengnsysGitInstaller() - installer.add_forgejo_repo(repo) + ret = git_forgejo_create_repo(repo) journal.send(f"Repository {repo} created", PRIORITY=journal.LOG_INFO, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") @@ -2879,49 +2846,33 @@ def git_add_sshkey(): if not "description" in data: data["description"] = "" - if not ("ssh_key" in data or "oglive" in data): - journal.send(f"Can't add SSH keys, either ssh_key or oglive is required", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") + if not ("ssh_key" in data ): + journal.send(f"Can't add SSH keys, ssh_key is required", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") return jsonify({"error" : "Parameters missing, specify ssh_key or oglive"}), 400 + status, content = git_forgejo_add_sshkey(data["ssh_key"], data["description"]) + message = "Result unrecognized" + success = False + httpcode = 500 - - module = _load_installer() - print(f"Got {module}") - OpengnsysGitInstaller = getattr(module, 'OpengnsysGitInstaller') - installer = OpengnsysGitInstaller() - - if "oglive" in data: - job_id = f"GitSshKey_{''.join(random.choice('0123456789abcdef') for char in range(8))}" - threading.Thread(target=git_add_sshkey_task, args=(data["oglive"], data["description"], job_id)).start() - return jsonify({ - "success": True, - "output": "Extracting key from ogLive...", - "job_id": job_id - }), 200 + if status == 200 or status == 201: + message = "SSH key added" + success = True + httpcode = 200 + elif status == 422: + message = "SSH key already existed" + success = True + httpcode = 200 else: - status, content = installer.add_forgejo_sshkey(data["ssh_key"], data["description"]) - message = "Result unrecognized" + message = "Unrecognized reply from forgejo" success = False - httpcode = 500 + httpcode = status + journal.send(f"Unrecognized reply from forgejo: code {status}, content {content}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - if status == 200 or status == 201: - message = "SSH key added" - success = True - httpcode = 200 - elif status == 422: - message = "SSH key already existed" - success = True - httpcode = 200 - else: - message = "Unrecognized reply from forgejo" - success = False - httpcode = status - journal.send(f"Unrecognized reply from forgejo: code {status}, content {content}", PRIORITY=journal.LOG_ERR, SYSLOG_IDENTIFIER="ogrepo-api_DEBUG") - - return jsonify({ - "success": success, - "output": message, - }), httpcode + return jsonify({ + "success": success, + "output": message, + }), httpcode # -------------------------------------------------------------------------------------------- diff --git a/debian/changelog b/debian/changelog index 369f622..ed0d97b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +ogrepository (1.1.4) UNRELEASED; urgency=medium + + * refs #2363 Reduce number of required dependencies + + -- OpenGnsys Tue, 30 Jun 2025 15:50:11 +0000 + +ogrepository (1.1.3) UNRELEASED; urgency=medium + + * refs #2317 Fix performance problems, remove stats + * refs #2298 Add commmits endpoint + + -- OpenGnsys Tue, 30 Jun 2025 15:50:11 +0000 + + ogrepository (1.1.2) UNRELEASED; urgency=medium * Fix path handling when there are no repositories From 73c0220ab3751d10d562cada2911df2ddb158330 Mon Sep 17 00:00:00 2001 From: Vadim Trochinsky Date: Tue, 1 Jul 2025 16:56:06 +0200 Subject: [PATCH 25/26] refs #2363 Fix breaking forgejo permissions --- debian/changelog | 6 ++++++ debian/ogrepository.postinst | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index ed0d97b..6bc3d13 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +ogrepository (1.1.5) UNRELEASED; urgency=medium + + * refs #2363 Fix breaking forgejo permissions + + -- OpenGnsys Tue, 1 Jul 2025 16:05:12 +0000 + ogrepository (1.1.4) UNRELEASED; urgency=medium * refs #2363 Reduce number of required dependencies diff --git a/debian/ogrepository.postinst b/debian/ogrepository.postinst index fff9a6a..f4b344e 100755 --- a/debian/ogrepository.postinst +++ b/debian/ogrepository.postinst @@ -92,7 +92,7 @@ done (echo "$SAMBA_PASS"; echo "$SAMBA_PASS") | smbpasswd -a $SAMBA_USER fi systemctl enable ogrepo-api - # Configure Repo + # Configure Repo cp /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg.tmpl /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg sed -i "s/SERVERIP/$OGREPO_IP/g" /opt/opengnsys/ogrepository/etc/ogAdmRepo.cfg @@ -128,6 +128,18 @@ chown -R opengnsys:www-data /opt/opengnsys/ogrepository chmod 755 /opt/opengnsys chmod 755 /opt/opengnsys/ogrepository/bin/* +# oggit -- these are from opengnsys-forgejo, and we're overwriting their +# permissions above. +# +# Check if the paths exist just in case the opengnsys-forgejo package +# stops being a dependency. +if [ -d "/opt/opengnsys/ogrepository/var/lib/forgejo/" ] ; then + chown -R oggit:oggit /opt/opengnsys/ogrepository/var/lib/forgejo/ +fi + +if [ -d "/opt/opengnsys/ogrepository/oggit/" ] ; then + chown -R oggit:oggit /opt/opengnsys/ogrepository/oggit/ +fi # Install http server stuff # Reiniciar servicios si es necesario From d048ee782c5698f57db3b06d436b90ccd8d1bde8 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 7 Jul 2025 09:56:49 +0200 Subject: [PATCH 26/26] refs #2371 hide commits that can't be restored --- api/repo_api.py | 30 +++++++++++++++++++++++++++++- api/swagger.yaml | 19 +++++++++++-------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/api/repo_api.py b/api/repo_api.py index c52ab95..705a2dc 100644 --- a/api/repo_api.py +++ b/api/repo_api.py @@ -2559,6 +2559,7 @@ def git_get_commits(repo, branch): max_commits = request.args.get('max_commits') or 100 skip_commits = request.args.get('skip') or 0 + filter_commits = (request.args.get('filter_commits') or 1) == 1 git_repo = git.Repo(repo_path) @@ -2580,14 +2581,41 @@ def git_get_commits(repo, branch): hash_to_tag[sha] = hash_to_tag[sha] + [ tag.name ] + first_commit = next(git_repo.iter_commits(reverse=True)) + commits = [] + git_repo.iter_commits() for com in git_repo.iter_commits(branch, max_count = max_commits, skip = skip_commits): tag_list = [] + # Initial check-ins contain only a .gitignore, used to test the repository is working, + # and do a force push to the destination. + # + # We don't want to show this one to the users by default, since it's not a functional + # restoration point. + # + # To make sure users can use git normally, we try to detect it in a slightly paranoid + # manner here. We want to avoid magic markers or to always skip the first commit, in + # case it might actually contain data. + skip_this_commit = False + + # Check: + # * First commit, and we want filtering + # * There's only one blob + # * The one file is .gitignore. + if com == first_commit and filter_commits: + if len(com.tree.blobs) == 1: + for file in com.tree.traverse(depth=1, visit_once=False): + if file.name == ".gitignore" and file.type == "blob": + skip_this_commit = True + + if skip_this_commit: + continue + + if com.hexsha in hash_to_tag: tag_list = hash_to_tag[com.hexsha] - commits = commits + [ { "hexsha" : com.hexsha, diff --git a/api/swagger.yaml b/api/swagger.yaml index 975bb29..ce258ea 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1982,9 +1982,6 @@ paths: Especificando **ssh_key** se agrega la cclave especificada directamente. - Especificando **oglive** se descarga si es necesario el .iso, se monta, y se extrae la clave. - Esta acción se hace en el fondo, y se devuelve un job_id. - tags: - "Git" parameters: @@ -2006,10 +2003,6 @@ paths: type: string example: "OgLive r20250518" required: False - oglive: - type: string - example: "https://ognproject.evlt.uma.es/oglive/ogLive-noble-6.8.0-31-generic-amd64-r20250518.cf13a6d_20250519.iso" - required: False responses: "200": description: "Exito" @@ -2270,7 +2263,12 @@ paths: get: summary: "Obtener lista de commits en una rama" description: | - Devuelve una lista de commits de Git + Devuelve una lista de commits de Git. + + **Filtrado de commits:** + Por defecto algunos commits que no son utiles para restaurar una imagen se ocultan. + Pasar filter_commits=0 para desactivar esta funcionalidad y ver todos los commits + del repositorio. tags: - "Git" parameters: @@ -2294,6 +2292,11 @@ paths: required: false type: int description: "Commits a saltar (para paginación)" + - name: filter_commits + in: query + required: false + type: int + description: "Si aplicar filtrado de commits. Por defecto 1." responses: "200": description: "Lista de commits"