CKN plugin: replace oracle plugin and fix event streaming to Patra via CKN broker

.github/workflows/docker-image.yml

@@ -588,7 +588,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push oracle_plugin
+      - name: Build and push ckn_plugin
         uses: docker/build-push-action@v5
         with:
           context: external_plugins/image_detecting_plugin
@@ -597,23 +597,11 @@ jobs:
           tags: tapis/image_detecting_plugin:${{ env.TRAPS_REL }}
           build-args: REL=${{ env.TRAPS_REL }}
 
-  power_measuring_plugin:
+  power_measuring_plugin_deps:
     runs-on: ubuntu-latest
     environment: main
     needs: camera_traps_py_313
     steps:
-      - name: Set TRAPS_REL
-        run: |
-          if [[ ${{ github.event_name == 'workflow_dispatch' }} == true ]]; then
-            TRAPS_REL=${{ inputs.release_tag }}
-          elif [[ ${{ github.event_name == 'push' }} == true ]]; then
-            TRAPS_REL=latest
-          elif [[ ${{ github.event_name == 'pull_request' }} == true ]]; then
-            TRAPS_REL=latest
-          else
-            TRAPS_REL=${{ github.event.release.tag_name }}
-          fi
-          echo "TRAPS_REL=$TRAPS_REL" >> "$GITHUB_ENV"
       - name: Checkout
         uses: actions/checkout@v4
       - name: Set up QEMU
@@ -641,6 +629,35 @@ jobs:
           file: external_plugins/power_measuring_plugin/Dockerfile-scaphandre
           platforms: linux/amd64,linux/arm64
           tags: tapis/scaphandre
+
+  power_measuring_plugin:
+    runs-on: ubuntu-latest
+    environment: main
+    needs: power_measuring_plugin_deps
+    steps:
+      - name: Set TRAPS_REL
+        run: |
+          if [[ ${{ github.event_name == 'workflow_dispatch' }} == true ]]; then
+            TRAPS_REL=${{ inputs.release_tag }}
+          elif [[ ${{ github.event_name == 'push' }} == true ]]; then
+            TRAPS_REL=latest
+          elif [[ ${{ github.event_name == 'pull_request' }} == true ]]; then
+            TRAPS_REL=latest
+          else
+            TRAPS_REL=${{ github.event.release.tag_name }}
+          fi
+          echo "TRAPS_REL=$TRAPS_REL" >> "$GITHUB_ENV"
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push power_measuring_plugin
         uses: docker/build-push-action@v5
         with:
@@ -650,7 +667,7 @@ jobs:
           tags: tapis/power_measuring_plugin_py:${{ env.TRAPS_REL }}
           build-args: REL=${{ env.TRAPS_REL }}
 
-  oracle_plugin:
+  ckn_plugin:
     runs-on: ubuntu-latest
     environment: main
     needs: camera_traps_py_313
@@ -678,13 +695,13 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push oracle_plugin
+      - name: Build and push ckn_plugin
         uses: docker/build-push-action@v5
         with:
-          context: external_plugins/oracle_plugin
+          context: external_plugins/ckn_plugin
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: tapis/oracle_plugin:${{ env.TRAPS_REL }}
+          tags: tapis/ckn_plugin:${{ env.TRAPS_REL }}
           build-args: REL=${{ env.TRAPS_REL }}
 
   detection_reporter_plugin:
@@ -715,7 +732,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push oracle_plugin
+      - name: Build and push ckn_plugin
         uses: docker/build-push-action@v5
         with:
           context: external_plugins/detection_reporter_plugin
@@ -752,7 +769,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push oracle_plugin
+      - name: Build and push ckn_plugin
         uses: docker/build-push-action@v5
         with:
           context: external_plugins/video_generating_plugin
@@ -822,11 +839,11 @@ jobs:
   trigger_tests:
     runs-on: ubuntu-latest
     environment: main
-    needs: [engine, image_scoring_plugin_server_py_313, image_scoring_plugin_ultralytics_py_313, image_scoring_plugin_yolov5_py_38, image_scoring_plugin_py_nano_38, image_generating_plugin, image_detecting_plugin, power_measuring_plugin, oracle_plugin, detection_reporter_plugin, video_generating_plugin, custom_install]
+    needs: [engine, image_scoring_plugin_server_py_313, image_scoring_plugin_ultralytics_py_313, image_scoring_plugin_yolov5_py_38, image_scoring_plugin_py_nano_38, image_generating_plugin, image_detecting_plugin, power_measuring_plugin_deps, power_measuring_plugin, ckn_plugin, detection_reporter_plugin, video_generating_plugin, custom_install]
     steps:
       - name: Check if tests are disabled
         run: |
-          if [[ ${{ github.event_name == 'workflow_dispatch' }} == true ]] && [ "${{ inputs.run_tests }}" == "false" ];
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [ "${{ inputs.run_tests }}" == "false" ];
           then
             echo "Tests are disabled"
             exit 1

Dockerfile

@@ -5,7 +5,7 @@
 # -------------------
 # First build phase: In this phase we do the build for release with an intermedidate layer that caches the dependencies
 # --------------------
-FROM rust:1.75 as builder
+FROM rust:1.90 as builder
 
 # install libzmq
 RUN USER=root apt-get update && apt-get install -y libzmq3-dev

Makefile

@@ -17,7 +17,7 @@
 #    Example: 'make build' builds all targets.
 
 clean:
-	cd releases/${TRAPS_REL}; rm -rf power_output_dir/*; rm -rf images_output_dir/*; rm -rf oracle_plugin_dir/*; 
+	cd releases/${TRAPS_REL}; rm -rf power_output_dir/*; rm -rf images_output_dir/*; rm -rf ckn_plugin_dir/*; 
 
 build-engine:
 	docker build -t tapis/camera_traps_engine:${TRAPS_REL} --build-arg TRAPS_REL=${TRAPS_REL} .
@@ -47,8 +47,8 @@ build-generating:
 build-power:
 	cd external_plugins/power_measuring_plugin/ && docker build -t tapis/powerjoular -f Dockerfile-powerjoular . && docker build -t tapis/scaphandre -f Dockerfile-scaphandre . && docker build -t tapis/power_measuring_plugin_py:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
 
-build-oracle:
-	cd external_plugins/oracle_plugin/ && docker build -t tapis/oracle_plugin:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
+build-ckn:
+	cd external_plugins/ckn_plugin/ && docker build -t tapis/ckn_plugin:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
 
 build-detection:
 	cd external_plugins/detection_reporter_plugin && docker build -t tapis/detection_reporter_plugin:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
@@ -59,18 +59,18 @@ build-detecting:
 build-video:
 	cd external_plugins/video_generating_plugin/ && docker build -t tapis/video_generating_plugin:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
 
-build-py-plugins: build-camerapy build-scoring-server build-generating build-power build-oracle build-detection build-detecting build-video
+build-py-plugins: build-camerapy build-scoring-server build-generating build-power build-ckn build-detection build-detecting build-video
 
 build-installer: 
 	cd installer && docker build -t tapis/camera-traps-installer:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../
 
 build: build-engine build-py-plugins build-installer
 
 tag: build
-	docker tag tapis/camera_traps_py:${TRAPS_REL} tapis/camera_traps_py & docker tag tapis/camera_traps_py_3.8:${TRAPS_REL} tapis/camera_traps_py_3.8 & docker tag tapis/image_scoring_plugin_py_3.8:${TRAPS_REL} tapis/image_scoring_plugin_py_3.8 & docker tag tapis/image_scoring_plugin_py_nano_3.8:${TRAPS_REL} tapis/image_scoring_plugin_py_nano_3.8 & docker tag tapis/image_generating_plugin_py:${TRAPS_REL} tapis/image_generating_plugin_py & docker tag tapis/power_measuring_plugin_py:${TRAPS_REL} tapis/power_measuring_plugin_py & docker tag tapis/camera_traps_engine:${TRAPS_REL} tapis/camera_traps_engine & docker tag tapis/oracle_plugin:${TRAPS_REL} tapis/oracle_plugin & docker tag tapis/camera-traps-installer:${TRAPS_REL} tapis/camera-traps-installer
+	docker tag tapis/camera_traps_py:${TRAPS_REL} tapis/camera_traps_py & docker tag tapis/camera_traps_py_3.8:${TRAPS_REL} tapis/camera_traps_py_3.8 & docker tag tapis/image_scoring_plugin_py_3.8:${TRAPS_REL} tapis/image_scoring_plugin_py_3.8 & docker tag tapis/image_scoring_plugin_py_nano_3.8:${TRAPS_REL} tapis/image_scoring_plugin_py_nano_3.8 & docker tag tapis/image_generating_plugin_py:${TRAPS_REL} tapis/image_generating_plugin_py & docker tag tapis/power_measuring_plugin_py:${TRAPS_REL} tapis/power_measuring_plugin_py & docker tag tapis/camera_traps_engine:${TRAPS_REL} tapis/camera_traps_engine & docker tag tapis/ckn_plugin:${TRAPS_REL} tapis/ckn_plugin & docker tag tapis/camera-traps-installer:${TRAPS_REL} tapis/camera-traps-installer
 
 push: tag
-	docker push tapis/camera_traps_py & docker push tapis/camera_traps_py:${TRAPS_REL} & docker push tapis/camera_traps_py_3.8 & docker push tapis/camera_traps_py_3.8:${TRAPS_REL} & docker push tapis/image_scoring_plugin_py_3.8 & docker push tapis/image_scoring_plugin_py_3.8:${TRAPS_REL} & docker push tapis/image_scoring_plugin_py_nano_3.8 & docker push tapis/image_scoring_plugin_py_nano_3.8:${TRAPS_REL} & docker push tapis/image_generating_plugin_py & docker push tapis/image_generating_plugin_py:${TRAPS_REL} & docker push tapis/power_measuring_plugin_py:${TRAPS_REL} & docker push tapis/oracle_plugin:${TRAPS_REL} & docker push tapis/camera_traps_engine & docker push tapis/camera_traps_engine:${TRAPS_REL} & docker push tapis/camera-traps-installer:${TRAPS_REL} & docker push tapis/camera-traps-installer
+	docker push tapis/camera_traps_py & docker push tapis/camera_traps_py:${TRAPS_REL} & docker push tapis/camera_traps_py_3.8 & docker push tapis/camera_traps_py_3.8:${TRAPS_REL} & docker push tapis/image_scoring_plugin_py_3.8 & docker push tapis/image_scoring_plugin_py_3.8:${TRAPS_REL} & docker push tapis/image_scoring_plugin_py_nano_3.8 & docker push tapis/image_scoring_plugin_py_nano_3.8:${TRAPS_REL} & docker push tapis/image_generating_plugin_py & docker push tapis/image_generating_plugin_py:${TRAPS_REL} & docker push tapis/power_measuring_plugin_py:${TRAPS_REL} & docker push tapis/ckn_plugin:${TRAPS_REL} & docker push tapis/camera_traps_engine & docker push tapis/camera_traps_engine:${TRAPS_REL} & docker push tapis/camera-traps-installer:${TRAPS_REL} & docker push tapis/camera-traps-installer
 
 push-all:
 	docker buildx build --platform linux/arm64,linux/amd64 --push -t tapis/camera_traps_engine:${TRAPS_REL} --build-arg TRAPS_REL=${TRAPS_REL} .
@@ -79,5 +79,5 @@ push-all:
 	cd external_plugins/image_scoring_plugin/ && docker buildx build --platform linux/arm64 -t tapis/image_scoring_plugin_py_nano_3.8:${TRAPS_REL} --build-arg REL=${TRAPS_REL} -f Dockerfile-3.8-nano .; cd ../..
 	cd external_plugins/image_generating_plugin/ && docker buildx build --platform linux/arm64,linux/amd64 --push -t tapis/image_generating_plugin_py:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
 	cd external_plugins/power_measuring_plugin/ && docker buildx build --platform linux/arm64,linux/amd64 --push -t tapis/power_measuring_plugin_py:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
-	cd external_plugins/oracle_plugin/ && docker buildx build --platform linux/arm64,linux/amd64 --push -t tapis/oracle_plugin:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
+	cd external_plugins/ckn_plugin/ && docker buildx build --platform linux/arm64,linux/amd64 --push -t tapis/ckn_plugin:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../..
 	cd installer && docker buildx build --platform linux/arm64,linux/amd64 --push -t tapis/camera-traps-installer:${TRAPS_REL} --build-arg REL=${TRAPS_REL} .; cd ../

README.md

@@ -36,7 +36,7 @@ In general, plugins can also depend on their own environment variables and/or co
 | detection_reporter_plugin| TRAPS_DETECTION_REPORTER_*    | /traps-detection.toml    |
 | image_store_plugin       | TRAPS_IMAGE_STORE_FILE        | ~/traps-image-store.toml |                                   |
 | power_measure_plugin     | TRAPS_POWER_LOG_PATH          | ~/logs                   |                                   |
-| oracle_monitor_plugin    | TRAPS_ORACLE_OUTPUT_PATH      | ~/output                 |                                   |
+| ckn_plugin               | TRAPS_CKN_OUTPUT_PATH         | ~/output                 |                                   |
 | integration tests        | TRAPS_INTEGRATION_CONFIG_FILE | ~/traps-integration.toml |                                   |
 | logger                   | TRAPS_LOG4RS_CONFIG_FILE      | resources/log4rs.yml     | Packaged with application         |
 
@@ -123,7 +123,7 @@ Camera-traps uses a [TOML](https://toml.io/en/) file to configure the internal a
 > "PluginTerminateEvent"<br>
 > ]<br>
 > [[plugins.external]]<br>
->   plugin_name = "ext_oracle_monitor_plugin"<br>
+>   plugin_name = "ext_ckn_plugin"<br>
 >   id = "6e153711-9823-4ee6-b608-58e2e801db51"<br>
 >  external_port = 6011<br>
 > subscriptions = [<br>

external_plugins/ckn_plugin/Dockerfile

@@ -0,0 +1,12 @@
+ARG REL=use-build-arg
+FROM tapis/camera_traps_py_3.13:$REL
+RUN find / -path /proc -prune -o -path /sys -prune -o -path /dev -prune -o -writable -print0 | xargs -0 -P $(nproc) -I {} chmod 0777 {} || true
+
+# Install confluent-kafka for Kafka streaming
+RUN pip install --no-cache-dir confluent-kafka
+
+ADD ckn_plugin.py /ckn_plugin.py
+ADD power_processor.py /power_processor.py
+RUN chmod -R 0777 /ckn_plugin.py /power_processor.py || true
+
+ENTRYPOINT [ "python", "-u", "/ckn_plugin.py" ]