TASK-5564 - Update data sources for CellBase 6.2

.github/workflows/develop.yml

@@ -8,21 +8,18 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-    with:
-      maven_opts: -DCELLBASE.WAR.NAME=cellbase -P default-config-test
-
   test:
     uses: ./.github/workflows/test-analysis.yml
-    needs: build
     secrets: inherit
+    with:
+      upload_artifact: true
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
     needs: test
     with:
       maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase
+      cache_key: ${{ needs.test.outputs.cache_key }}
     secrets: inherit
 
   deploy-docker:

.github/workflows/manual-test.yml

@@ -12,11 +12,26 @@ on:
         description: 'The process executes all tests even if some fail.'
         default: false
         required: false
+      upload_artifact:
+        type: boolean
+        description: 'Upload build artifact.'
+        default: false
+        required: false
 
 jobs:
+  build:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      needs_hadoop_preparation: false
+      maven_opts: -Dcheckstyle.skip
+      upload_artifact: ${{ inputs.upload_artifact }}
+      dependency_repos: "java-common-libs,biodata"
+    secrets: inherit
+
   test:
     name: JUnit Test
-    runs-on: ubuntu-22.04
+    needs: build
+    runs-on: ${{ vars.UBUNTU_VERSION }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -34,14 +49,13 @@ jobs:
           chmod +x ./kubectl
           echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf 
           ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27017:27017 --kubeconfig ./admin.conf &
-      - name: Install dependencies branches
-        run: |
-          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
-            chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
-          else
-             echo "./.github/workflows/scripts/get_same_branch.sh does not exist."
-          fi
+      - name: Cache local Maven repository
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ needs.build.outputs.cache_key }}
+          ## Force cache hit to avoid testing with incomplete dependencies
+          fail-on-cache-miss: true
       - name: Test and Analyze
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any

.github/workflows/pull-request-approved.yml

@@ -1,15 +1,13 @@
 name: Pull request approve workflow
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
 
 on:
   pull_request_review:
     types: [ submitted ]
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-
   test:
-    name: "Test analysis"
     uses: ./.github/workflows/test-analysis.yml
-    needs: build
     secrets: inherit
+    with:
+      upload_artifact: false

.github/workflows/pull-request-merge.yml

@@ -9,8 +9,9 @@ on:
       - closed
 
 jobs:
-  delete-docker:
-    uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop
+  call-delete-docker:
+    name: Call Reusable Delete Docker Workflow
+    uses: opencb/cellbase/.github/workflows/reusable-delete-docker.yml@develop
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
+      task: ${{ github.head_ref }}
     secrets: inherit

.github/workflows/release.yml

@@ -11,12 +11,15 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
       maven_opts: -DCELLBASE.WAR.NAME=cellbase
+      dependency_repos: "java-common-libs,biodata"
+      upload_artifact: true
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
     needs: build
     with:
       maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase
+      cache_key: ${{ needs.build.outputs.cache_key }}
     secrets: inherit
 
   deploy-docker:

.github/workflows/scripts/get-xetabase-branch.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Function to calculate the corresponding branch of Xetabase project
+get_xetabase_branch() {
+  # Input parameter (branch name)
+  input_branch="$1"
+
+  # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
+  if [[ $input_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
+      echo $input_branch;
+      return 0;
+    fi
+  fi
+
+  # Check if the branch name is "develop" in that case return the same branch name
+  if [[ "$input_branch" == "develop" ]]; then
+    echo "develop"
+    return 0
+  fi
+
+  # Check if the branch name starts with "release-" and follows the patterns "release-a.x.x" or "release-a.b.x"
+  if [[ "$input_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
+    # Extract the MAJOR part of the branch name
+    MAJOR=${BASH_REMATCH[1]}
+    # Calculate the XETABASE_MAJOR by subtracting 4 from MAJOR of cellbase
+    XETABASE_MAJOR=$((MAJOR - 4))
+    # Check if the XETABASE_MAJOR is negative
+    if (( XETABASE_MAJOR < 0 )); then
+      echo "Error: 'MAJOR' digit after subtraction results in a negative number."
+      return 1
+    fi
+    # Construct and echo the new branch name
+    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    return 0
+  fi
+
+  # If the branch name does not match any of the expected patterns
+  echo "Error: The branch name is not correct."
+  return 1
+}
+
+# Check if the script receives exactly one argument
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <branch-name>"
+  exit 1
+fi
+
+# Call the function with the input branch name
+get_xetabase_branch "$1"

.github/workflows/task.yml

@@ -1,4 +1,4 @@
-name: CellBase Develop workflow
+name: CellBase task workflow
 
 on:
   push:
@@ -7,19 +7,15 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-    with:
-      maven_opts: -DCELLBASE.WAR.NAME=cellbase -P default-config-test
-
   test:
     uses: ./.github/workflows/test-analysis.yml
-    needs: build
     secrets: inherit
+    with:
+      upload_artifact: true
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: test
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base --tag ${{ github.ref_name }}
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,builder --tag ${{ github.ref_name }}
     secrets: inherit

.github/workflows/test-analysis.yml

@@ -2,14 +2,34 @@ name: Build and test the project
 
 on:
   workflow_call:
+    inputs:
+      upload_artifact:
+        type: boolean
+        required: false
+        default: false
+    outputs:
+      cache_key:
+        description: "Cache key used for Maven repository"
+        value: ${{ jobs.build.outputs.cache_key }}
     secrets:
       SONAR_TOKEN:
         required: true
 
 jobs:
+
+  build:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      needs_hadoop_preparation: false
+      maven_opts: -Dcheckstyle.skip -DCELLBASE.WAR.NAME=cellbase
+      upload_artifact: ${{ inputs.upload_artifact }}
+      dependency_repos: "java-common-libs,biodata"
+    secrets: inherit
+
   test:
     name: Test and push Sonar analysis
-    runs-on: ubuntu-22.04
+    needs: build
+    runs-on: ${{ vars.UBUNTU_VERSION }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -26,14 +46,13 @@ jobs:
           chmod +x ./kubectl
           echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf 
           ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27017:27017 --kubeconfig ./admin.conf &
-      - name: Install dependencies branches
-        run: |
-          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
-            chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
-          else
-             echo "./.github/workflows/scripts/get_same_branch.sh does not exist."
-          fi
+      - name: Cache local Maven repository
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ needs.build.outputs.cache_key }}
+          ## Force cache hit to avoid testing with incomplete dependencies
+          fail-on-cache-miss: true
       - name: Test and Analyze
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any

cellbase-app/app/cloud/docker/cellbase-builder/Dockerfile

@@ -11,7 +11,7 @@ LABEL org.label-schema.vendor="OpenCB" \
 ## We need to be root to install dependencies
 USER root
 RUN apt-get update -y && \
-    apt-get install -y git default-mysql-client libjson-perl libdbi-perl libdbd-mysql-perl libdbd-mysql-perl libtry-tiny-perl && \
+    apt-get install -y git default-mysql-client libjson-perl libdbi-perl libdbd-mysql-perl libdbd-mysql-perl libtry-tiny-perl libxml-simple-perl liblog-log4perl-perl libxml-parser-perl libxml-dom-perl && \
     mkdir /opt/ensembl && chown cellbase:cellbase /opt/ensembl && \
     rm -rf /var/lib/apt/lists/*
 
@@ -26,6 +26,10 @@ RUN cd /opt/ensembl && \
     git clone https://github.com/Ensembl/ensembl-variation.git && \
     git clone https://github.com/Ensembl/ensembl-funcgen.git && \
     git clone https://github.com/Ensembl/ensembl-compara.git && \
-    git clone https://github.com/Ensembl/ensembl-io.git
+    git clone https://github.com/Ensembl/ensembl-io.git && \
+    git clone --branch cvs/release-0_7 https://github.com/biomart/biomart-perl
 
-ENV PERL5LIB=$PERL5LIB:/opt/ensembl/bioperl-live:/opt/ensembl/ensembl/modules:/opt/ensembl/ensembl-variation/modules:/opt/ensembl/ensembl-funcgen/modules:/opt/ensembl/ensembl-compara/modules:/opt/ensembl/lib/perl/5.18.2:/opt/cellbase/scripts/ensembl-scripts
+## Give writting permissions to allow the script ensembl_canonical.pl to create sub-folder for cache purposes
+RUN chmod -R 777 /opt/cellbase/scripts/ensembl-scripts/
+
+ENV PERL5LIB=$PERL5LIB:/opt/ensembl/bioperl-live:/opt/ensembl/ensembl/modules:/opt/ensembl/ensembl-variation/modules:/opt/ensembl/ensembl-funcgen/modules:/opt/ensembl/ensembl-compara/modules:/opt/ensembl/lib/perl/5.18.2:/opt/cellbase/scripts/ensembl-scripts:/opt/ensembl/biomart-perl/lib

cellbase-app/app/cloud/docker/docker-build.py

@@ -17,6 +17,7 @@
 parser.add_argument('--build-folder', help="the location of the build folder, if not default location")
 parser.add_argument('--username', help="credentials for dockerhub (REQUIRED if deleting from DockerHub)")
 parser.add_argument('--password', help="credentials for dockerhub (REQUIRED if deleting from DockerHub)")
+parser.add_argument('--org', help="Docker organization", default="opencb")
 
 ## Some ANSI colors to print shell output
 shell_colors = {
@@ -62,31 +63,31 @@ def build():
     print_header('Building docker images: ' + ', '.join(images))
     for image in images:
         print()
-        print(shell_colors['blue'] + "Building opencb/cellbase-" + image + ":" + tag + " ..." + shell_colors['reset'])
+        print(shell_colors['blue'] + "Building " + org + "/cellbase-" + image + ":" + tag + " ..." + shell_colors['reset'])
         if image == "base":
-            run("docker build -t opencb/cellbase-" + image + ":" + tag + " -f " + build_folder + "/cloud/docker/cellbase-" + image + "/Dockerfile " + build_folder)
+            run("docker build -t " + org + "/cellbase-" + image + ":" + tag + " -f " + build_folder + "/cloud/docker/cellbase-" + image + "/Dockerfile " + build_folder)
         else:
-            run("docker build -t opencb/cellbase-" + image + ":" + tag + " -f " + build_folder + "/cloud/docker/cellbase-" + image + "/Dockerfile --build-arg TAG=" + tag + " " + build_folder)
+            run("docker build -t " + org + "/cellbase-" + image + ":" + tag + " -f " + build_folder + "/cloud/docker/cellbase-" + image + "/Dockerfile --build-arg TAG=" + tag + " " + build_folder)
 
 
 def tag_latest(image):
-    latest_tag = os.popen(("curl -s https://registry.hub.docker.com/v1/repositories/opencb/cellbase-" + image + "/tags"
+    latest_tag = os.popen(("curl -s https://registry.hub.docker.com/v1/repositories/" + org + "/cellbase-" + image + "/tags"
                            + " | jq -r .[].name"
                            + " | grep -v latest"
                            + " | sort -h"
                            + " | head"))
     if tag >= latest_tag.read():
-        print(shell_colors['blue'] + "Pushing opencb/cellbase-" + image + ":latest" + shell_colors['reset'])
-        run("docker tag opencb/cellbase-" + image + ":" + tag + " opencb/cellbase-" + image + ":latest")
-        run("docker push opencb/cellbase-" + image + ":latest")
+        print(shell_colors['blue'] + "Pushing " + org + "/cellbase-" + image + ":latest" + shell_colors['reset'])
+        run("docker tag " + org + "/cellbase-" + image + ":" + tag + " " + org + "/cellbase-" + image + ":latest")
+        run("docker push " + org + "/cellbase-" + image + ":latest")
 
 
 def push():
     print_header('Pushing to DockerHub: ' + ', '.join(images))
     for i in images:
         print()
-        print(shell_colors['blue'] + "Pushing opencb/cellbase-" + i + ":" + tag + " ..." + shell_colors['reset'])
-        run("docker push opencb/cellbase-" + i + ":" + tag)
+        print(shell_colors['blue'] + "Pushing " + org + "/cellbase-" + i + ":" + tag + " ..." + shell_colors['reset'])
+        run("docker push " + org + "/cellbase-" + i + ":" + tag)
         tag_latest(i)
 
 
@@ -104,11 +105,11 @@ def delete():
         error("dockerhub login failed")
     for i in images:
         print()
-        print(shell_colors['blue'] + 'Deleting image on Docker hub for opencb/cellbase-' + i + ':' + tag + shell_colors['reset'])
+        print(shell_colors['blue'] + 'Deleting image on Docker hub for ' + org + '/cellbase-' + i + ':' + tag + shell_colors['reset'])
         headers = {
             'Authorization': 'JWT ' + json_response["token"]
         }
-        requests.delete('https://hub.docker.com/v2/repositories/opencb/cellbase-' + i + '/tags/' + tag + '/', headers=headers)
+        requests.delete('https://hub.docker.com/v2/repositories/' + org + '/cellbase-' + i + '/tags/' + tag + '/', headers=headers)
 
 
 ## Parse command-line parameters and init basedir, tag and build_folder
@@ -144,6 +145,11 @@ def delete():
 else:
     images = args.images.split(",")
 
+# 5. Set docker org to default value if not set
+if args.org is not None:
+    org = args.org
+else:
+    org = "opencb"
 
 ## Execute the action
 if args.action == "build":