Testing

database/createDatabases.ps1

@@ -2,13 +2,13 @@
 
 <#   
 .SYNOPSIS   
-   Create Databases Required for AIFabric
+   Create Databases Required for AICenter
 .DESCRIPTION 
-   Create Databases required by AIFabric i.e ai_helper, ai_pkgmanager, ai_deployer, ai_trainer & ai_appmanager & the script also creates an user which has db_owner privileges on all these 5 databases 
+   Create Databases required by AICenter i.e ai_helper, ai_pkgmanager, ai_deployer, ai_trainer & ai_appmanager & the script also creates an user which has db_owner privileges on all these 5 databases 
    and the username and password that are generated are both logged to console as well as stored as an file in the current directory from where the script execution is triggered.
 .NOTES   
     Name: ./createDatabases.ps1
-    Author: AIFabric Team
+    Author: AICenter Team
     Pre-Requisites: script has to be executed throuh powershell in Administrator mode & before running script set execution policy to RemoteSigned by running "Set-ExecutionPolicy RemoteSigned"
 .EXAMPLE 
     If SQL Server can be accessed through Windows Authentication then:

gpu/attach_gpu_drivers.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+function edit_daemon_json(){
+    echo "################## Updating docker configuration ######################"
+    sudo bash -c '
+echo \
+"{
+    \"default-runtime\": \"nvidia\",
+    \"exec-opts\": [\"native.cgroupdriver=systemd\"],
+    \"runtimes\": {
+        \"nvidia\": {
+            \"path\": \"/usr/bin/nvidia-container-runtime\",
+            \"runtimeArgs\": []
+        }
+    }
+}" > /etc/docker/daemon.json'
+
+}
+
+function kubernetes_cluster_up() {
+    count=0
+    swapoff -a
+    while [ $count -lt 50 ]; do
+        sudo chmod +r /etc/kubernetes/admin.conf
+        export KUBECONFIG=/etc/kubernetes/admin.conf
+        result=$(kubectl get nodes| grep master)
+        if [[ "$result" == *"master"* ]]; then
+            echo "Kubernetes up after " $((count * 5)) "seconds"
+            break
+        else
+            echo "Kubernetes not up, retry : " $count
+            count=$(( $count + 1 ))
+            sleep 5
+        fi
+    done
+
+    if [ $count == 50 ]; then
+        echo "Kubernetes Failed to come up"
+        exit
+    fi
+}
+
+function validate_gpu_updated() {
+    count=0
+    while [ $count -lt 50 ]; do
+        result=$(kubectl describe nodes| grep nvidia.com/gpu)
+        if [[ "$result" == *"nvidia.com/gpu"* ]]; then
+            echo $result
+            echo "Node gpu info updated after " $((count * 5)) "seconds"
+            echo "##################### Successfully installed GPU #########################"
+            break
+        else
+            echo "kubectl gpu info not updated, retry : " $count
+            count=$(( $count + 1 ))
+            sleep 5
+        fi
+    done
+
+    if [ $count == 50 ]; then
+        echo "################## Failed to install gpu ####################"
+        swapoff -a
+        exit
+    fi
+}
+
+function restart_docker() {
+    sudo pkill -SIGHUP dockerd
+    sudo systemctl restart docker
+    count=0
+    while [ $count -lt 50 ]; do
+        result=$(sudo systemctl status docker| grep running)
+        if [[ "$result" == *"running"* ]]; then
+            echo "docker is up " $((count * 5)) "seconds"
+            break
+        else
+            echo "docker is not up, retry : " $count
+            count=$(( $count + 1 ))
+            sleep 5
+        fi
+    done
+
+    if [ $count == 50 ]; then
+        echo "Docker Failed to come up"
+        swapoff -a
+        exit
+    fi
+}
+echo "#################################### Ecko Shutdown #######################################"
+sudo /opt/ekco/shutdown.sh
+
+echo "################################# Attach GPU Driver #####################################"
+# edit json
+edit_daemon_json
+echo "################################# Restarting docker #####################################"
+restart_docker
+sleep 2
+# This is required because when kubeadm init start kubelet, its check docker cgroup driver and uses cgroupfs
+# in case of discrepancy. So we have to change this driver and restart kubelet again
+echo "################################# Restarting kubelet #####################################"
+sudo sed -i 's/cgroup-driver=cgroupfs/cgroup-driver=systemd/' /var/lib/kubelet/kubeadm-flags.env
+sudo systemctl restart kubelet
+sleep 10
+kubernetes_cluster_up
+kubectl apply -f nvidia-device-plugin.yaml
+validate_gpu_updated
+
+echo "########################## Uncordon Node #######################################"
+kubectl uncordon $(hostname | tr '[:upper:]' '[:lower:]')
+
+echo "################ GPU driver installation successful ###################"

gpu/nvidia-device-plugin.yaml

@@ -0,0 +1,61 @@
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-device-plugin-daemonset
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: nvidia-device-plugin-ds
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      # This annotation is deprecated. Kept here for backward compatibility
+      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        name: nvidia-device-plugin-ds
+    spec:
+      tolerations:
+      # This toleration is deprecated. Kept here for backward compatibility
+      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: nvidia.com/gpu
+        operator: Exists
+        effect: NoSchedule
+      # Mark this pod as a critical add-on; when enabled, the critical add-on
+      # scheduler reserves resources for critical add-on pods so that they can
+      # be rescheduled after a failure.
+      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+      priorityClassName: "system-node-critical"
+      containers:
+      - image: nvidia/k8s-device-plugin:1.0.0-beta6
+        name: nvidia-device-plugin-ctr
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+        volumeMounts:
+          - name: device-plugin
+            mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+        - name: device-plugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins

language_version_metadata/PYTHON36_CV__3__metadata.json

@@ -0,0 +1,20 @@
+[{
+        "version": 3,
+        "imageType": "SERVING",
+        "processor": "CPU",
+        "mlPackageLanguage": "PYTHON36_CV",
+        "languageGroup": "CV",
+        "baseImage": "python36cv:22.4.0-3",
+        "runtimeImage": "",
+        "displayName": "Python 36 CV"
+},
+{
+        "version": 3,
+        "imageType": "SERVING",
+        "processor": "GPU",
+        "mlPackageLanguage": "PYTHON36_CV",
+        "languageGroup": "CV",
+        "baseImage": "python36cv:22.4.0-3",
+        "runtimeImage": "",
+        "displayName": "Python 36 CV"
+}]

language_version_metadata/Python36_CV__2__metadata.json

@@ -0,0 +1,20 @@
+[{
+        "version": 2,
+        "imageType": "SERVING",
+        "processor": "CPU",
+        "mlPackageLanguage": "PYTHON36_CV",
+        "languageGroup": "CV",
+        "baseImage": "python36cv:v21.10.0-2",
+        "runtimeImage": "",
+        "displayName": "Python 36 CV"
+},
+{
+        "version": 2,
+        "imageType": "SERVING",
+        "processor": "GPU",
+        "mlPackageLanguage": "PYTHON36_CV",
+        "languageGroup": "CV",
+        "baseImage": "python36cv:v21.10.0-2",
+        "runtimeImage": "",
+        "displayName": "Python 36 CV"
+}]

metadata/1040ScheduleC__1__metadata.json

@@ -0,0 +1,34 @@
+{
+    "name":  "form_1040ScheduleC",
+    "retrainable":  true,
+    "gpu":  0,
+    "processorType":  "CPU",
+    "description":  "Machine Learning model (available in Preview) for extracting commonly occurring data points from 1040ScheduleC. Please see more details including supported languages and sample documents in the official documentation page for Pre-trained Out of the Box models: https://docs.uipath.com/document-understanding/automation-cloud/latest/user-guide/out-of-the-box-pre-trained-ml-packages.",
+    "inputDescription":  "ML Skills deployed using this ML Package can be integrated into RPA workflows using the Machine Learning Extractor activity in the Official feed or the MLModel activity in Connect feed. File formats accepted include pdf, tiff, jpg or png files. The activities also require an API Key input which you need to obtain from your UiPath Automation Cloud account, in the Licenses -\u003e Other Services view.",
+    "outputDescription":  "Please refer to the documentation of the Activity used to query the ML Skill.",
+    "changeLog":  "",
+    "cpu":  1,
+    "inputType":  "JSON",
+    "displayName":  "1040ScheduleC",
+    "memory":  0,
+    "mlPackageLanguage":  "PYTHON37_DU",
+    "projectId":  "[project-id]",
+    "stagingUri":  "[staging-uri]",
+    "projectName":  "UiPath Document Understanding",
+    "projectDescription":  "UiPath models to classify and extract information from images and pdfs.",
+    "tenantName":  "UiPath",
+    "minAIFabricVersion":  "22.10",
+    "languageVersion":  4,
+    "version":  1,
+    "customVersion":  "23.10.2",
+    "imagePath":  "du-semistructured:v23.10.2-rc22",
+    "parametersFileJSON":  "{\"params\": [{\"name\": \"auto_retraining\",\"type\": [\"PIPELINE\"],\"value\": \"False\",\"help\": \"Enable this option when the Pipeline you are creating is a scheduled automatic pipeline, synchronized with automatic exports from Data Manager.\",\"options\": [\"True\",\"False\"]}],\"probes\": [{\"probe\": \"startup\",\"initialDelaySeconds\": \"120\",\"periodSeconds\": \"10\",\"timeoutSeconds\": \"2\",\"successThreshold\": \"1\",\"failureThreshold\": \"60\"}]}",
+    "settings":  [
+                     {
+                         "key":  "package_type",
+                         "type":  "STRING",
+                         "value":  "1040_schedule_c"
+                     }
+                 ]
+}
+

metadata/1040ScheduleC__2__metadata.json

@@ -0,0 +1,34 @@
+{
+    "name":  "form_1040ScheduleC",
+    "retrainable":  true,
+    "gpu":  0,
+    "processorType":  "CPU",
+    "description":  "Machine Learning model (available in Preview) for extracting commonly occurring data points from 1040ScheduleC. Please see more details including supported languages and sample documents in the official documentation page for Pre-trained Out of the Box models: https://docs.uipath.com/document-understanding/automation-cloud/latest/user-guide/out-of-the-box-pre-trained-ml-packages.",
+    "inputDescription":  "ML Skills deployed using this ML Package can be integrated into RPA workflows using the Machine Learning Extractor activity in the Official feed or the MLModel activity in Connect feed. File formats accepted include pdf, tiff, jpg or png files. The activities also require an API Key input which you need to obtain from your UiPath Automation Cloud account, in the Licenses -\u003e Other Services view.",
+    "outputDescription":  "Please refer to the documentation of the Activity used to query the ML Skill.",
+    "changeLog":  "",
+    "cpu":  1,
+    "inputType":  "JSON",
+    "displayName":  "1040ScheduleC",
+    "memory":  0,
+    "mlPackageLanguage":  "PYTHON37_DU",
+    "projectId":  "[project-id]",
+    "stagingUri":  "[staging-uri]",
+    "projectName":  "UiPath Document Understanding",
+    "projectDescription":  "UiPath models to classify and extract information from images and pdfs.",
+    "tenantName":  "UiPath",
+    "minAIFabricVersion":  "22.10",
+    "languageVersion":  4,
+    "version":  2,
+    "customVersion":  "23.10.3",
+    "imagePath":  "du-semistructured:v23.10.3-rc04",
+    "parametersFileJSON":  "{\"params\": [{\"name\": \"auto_retraining\",\"type\": [\"PIPELINE\"],\"value\": \"False\",\"help\": \"Enable this option when the Pipeline you are creating is a scheduled automatic pipeline, synchronized with automatic exports from Data Manager.\",\"options\": [\"True\",\"False\"]}],\"probes\": [{\"probe\": \"startup\",\"initialDelaySeconds\": \"120\",\"periodSeconds\": \"10\",\"timeoutSeconds\": \"2\",\"successThreshold\": \"1\",\"failureThreshold\": \"60\"}]}",
+    "settings":  [
+                     {
+                         "key":  "package_type",
+                         "type":  "STRING",
+                         "value":  "1040_schedule_c"
+                     }
+                 ]
+}
+

metadata/1040ScheduleC__3__metadata.json

@@ -0,0 +1,34 @@
+{
+    "name":  "form_1040ScheduleC",
+    "retrainable":  true,
+    "gpu":  0,
+    "processorType":  "CPU",
+    "description":  "Machine Learning model (available in Preview) for extracting commonly occurring data points from 1040ScheduleC. Please see more details including supported languages and sample documents in the official documentation page for Pre-trained Out of the Box models: https://docs.uipath.com/document-understanding/automation-cloud/latest/user-guide/out-of-the-box-pre-trained-ml-packages.",
+    "inputDescription":  "ML Skills deployed using this ML Package can be integrated into RPA workflows using the Machine Learning Extractor activity in the Official feed or the MLModel activity in Connect feed. File formats accepted include pdf, tiff, jpg or png files. The activities also require an API Key input which you need to obtain from your UiPath Automation Cloud account, in the Licenses -\u003e Other Services view.",
+    "outputDescription":  "Please refer to the documentation of the Activity used to query the ML Skill.",
+    "changeLog":  "",
+    "cpu":  1,
+    "inputType":  "JSON",
+    "displayName":  "1040ScheduleC",
+    "memory":  0,
+    "mlPackageLanguage":  "PYTHON37_DU",
+    "projectId":  "[project-id]",
+    "stagingUri":  "[staging-uri]",
+    "projectName":  "UiPath Document Understanding",
+    "projectDescription":  "UiPath models to classify and extract information from images and pdfs.",
+    "tenantName":  "UiPath",
+    "minAIFabricVersion":  "22.10",
+    "languageVersion":  4,
+    "version":  3,
+    "customVersion":  "23.10.4",
+    "imagePath":  "du-semistructured:v23.10.4-rc125",
+    "parametersFileJSON":  "{\"params\": [{\"name\": \"auto_retraining\",\"type\": [\"PIPELINE\"],\"value\": \"False\",\"help\": \"Enable this option when the Pipeline you are creating is a scheduled automatic pipeline, synchronized with automatic exports from Data Manager.\",\"options\": [\"True\",\"False\"]}],\"probes\": [{\"probe\": \"startup\",\"initialDelaySeconds\": \"120\",\"periodSeconds\": \"10\",\"timeoutSeconds\": \"2\",\"successThreshold\": \"1\",\"failureThreshold\": \"60\"}]}",
+    "settings":  [
+                     {
+                         "key":  "package_type",
+                         "type":  "STRING",
+                         "value":  "1040_schedule_c"
+                     }
+                 ]
+}
+

metadata/1040ScheduleC__4__metadata.json

@@ -0,0 +1,34 @@
+{
+    "name":  "form_1040ScheduleC",
+    "retrainable":  true,
+    "gpu":  0,
+    "processorType":  "CPU",
+    "description":  "Machine Learning model (available in Preview) for extracting commonly occurring data points from 1040ScheduleC. Please see more details including supported languages and sample documents in the official documentation page for Pre-trained Out of the Box models: https://docs.uipath.com/document-understanding/automation-cloud/latest/user-guide/out-of-the-box-pre-trained-ml-packages.",
+    "inputDescription":  "ML Skills deployed using this ML Package can be integrated into RPA workflows using the Machine Learning Extractor activity in the Official feed or the MLModel activity in Connect feed. File formats accepted include pdf, tiff, jpg or png files. The activities also require an API Key input which you need to obtain from your UiPath Automation Cloud account, in the Licenses -\u003e Other Services view.",
+    "outputDescription":  "Please refer to the documentation of the Activity used to query the ML Skill.",
+    "changeLog":  "",
+    "cpu":  1,
+    "inputType":  "JSON",
+    "displayName":  "1040ScheduleC",
+    "memory":  0,
+    "mlPackageLanguage":  "PYTHON37_DU",
+    "projectId":  "[project-id]",
+    "stagingUri":  "[staging-uri]",
+    "projectName":  "UiPath Document Understanding",
+    "projectDescription":  "UiPath models to classify and extract information from images and pdfs.",
+    "tenantName":  "UiPath",
+    "minAIFabricVersion":  "22.10",
+    "languageVersion":  4,
+    "version":  4,
+    "customVersion":  "23.10.5",
+    "imagePath":  "du-semistructured:v23.10-rc28",
+    "parametersFileJSON":  "{\"params\": [{\"name\": \"auto_retraining\",\"type\": [\"PIPELINE\"],\"value\": \"False\",\"help\": \"Enable this option when the Pipeline you are creating is a scheduled automatic pipeline, synchronized with automatic exports from Data Manager.\",\"options\": [\"True\",\"False\"]}],\"probes\": [{\"probe\": \"startup\",\"initialDelaySeconds\": \"120\",\"periodSeconds\": \"10\",\"timeoutSeconds\": \"2\",\"successThreshold\": \"1\",\"failureThreshold\": \"60\"}]}",
+    "settings":  [
+                     {
+                         "key":  "package_type",
+                         "type":  "STRING",
+                         "value":  "1040_schedule_c"
+                     }
+                 ]
+}
+