add full url to metadata

savantly-net · Mar 22, 2024 · c3e2109 · c3e2109
1 parent c53702e
commit c3e2109
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 6 deletions.
diff --git a/.github/workflows/upload-docs.yml b/.github/workflows/upload-docs.yml
@@ -0,0 +1,29 @@
+name: Upload Files to Mainbot
+
+# I know its crazy, but this plugin uses itself!! =)
+
+on:
+  workflow_dispatch: {} # Manually trigger the workflow
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'README.md'
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Upload Files
+        uses: savantly-net/mainbot-github-action@main
+        with:
+          glob-patterns: '*.md'  # Set your file pattern here
+          namespace: '/admin'
+          api-url: 'https://backstage-mainbot.savantly.apps.savantly.cloud'
+          client-id: ${{ secrets.MAINBOT_CLIENT_ID }}
+          client-secret: ${{ secrets.MAINBOT_CLIENT_SECRET }}
+          token-endpoint: https://oidc.apps.savantly.cloud/realms/savantly/protocol/openid-connect/token
diff --git a/README.md b/README.md
@@ -25,6 +25,10 @@ This GitHub Action enables you to automatically upload files from your repositor
 
 **Optional** The client secret for OAuth authentication, if required by your API.  
 
+### `token-endpoint`
+
+**Optional** The token endpoint for OAuth authentication, if required by your API.
+
 
 
 ## Features
@@ -52,9 +56,12 @@ This GitHub Action enables you to automatically upload files from your repositor
    name: Upload Files to API
 
    on:
+     workflow_dispatch: {} # Enable manual triggering
      push:
        branches:
          - main  # Set your branch here
+       paths:
+         - 'docs/**'  # Set your docs folder path here
 
    jobs:
      upload:
@@ -68,9 +75,10 @@ This GitHub Action enables you to automatically upload files from your repositor
            with:
              glob-patterns: '**/*.txt'  # Set your file pattern here
              namespace: '/admin'
-             api-url: 'https://mainbot.savantly.net/api/document/add'
-             client-id: ${{ secrets.CLIENT_ID }}
-             client-secret: ${{ secrets.CLIENT_SECRET }}
+             api-url: 'https://mainbot.savantly.net' # Set your Mainbot API endpoint here
+             client-id: ${{ secrets.CLIENT_ID }} # Set your client ID here
+             client-secret: ${{ secrets.CLIENT_SECRET }} # Set your client secret here
+             token-endpoint: 'https://mainbot.savantly.net/oauth/token' # Set your token endpoint here
    ```
 
 3. **Set GitHub Secrets**

diff --git a/index.ts b/index.ts
@@ -43,6 +43,9 @@ async function run() {
       info("No client id or client secret provided, uploading without token");
     }
 
+    var baseUrl = context.payload.repository?.html_url || "";
+    var baseFileUrl = baseUrl + "/blob/" + context.sha + "/";
+
     uploadFiles({
       namespace,
       globPatterns,
@@ -53,6 +56,7 @@ async function run() {
         repo: context.repo.repo,
         owner: context.repo.owner,
       },
+      baseFileUrl: baseFileUrl,
     });
   } catch (error) {
     logError(JSON.stringify(error));

diff --git a/lib/upload.ts b/lib/upload.ts
@@ -71,6 +71,7 @@ export interface UploadFilesOptions {
   apiUrl: string;
   token?: string;
   metadata: Record<string, string>;
+  baseFileUrl: string;
 }
 
 async function uploadFiles({
@@ -79,6 +80,7 @@ async function uploadFiles({
   apiUrl,
   token,
   metadata,
+  baseFileUrl
 }: UploadFilesOptions): Promise<void> {
   info(
     `uploadFiles: namespace: ${namespace}, globPatterns: ${globPatterns}, apiUrl: ${apiUrl}`
@@ -90,21 +92,22 @@ async function uploadFiles({
       for (const file of files) {
         try {
           const fileContent = fs.readFileSync(file, "utf8");
-          const relativePath = path.relative(process.cwd(), file);
+          const fullUrl = `${baseFileUrl}/${file}`;
           const response = await postDocument({
             namespace,
             documentText: fileContent,
             metadata: {
               ...metadata,
-              path: relativePath,
+              path: file,
+              url: fullUrl,
             },
             apiUrl,
             token,
           });
 
           if (response) {
             info(
-              `File uploaded successfully. ${relativePath} created vectors: ${file}`
+              `File uploaded successfully. ${file} created vectors: ${file}`
             );
           }
         } catch (error) {