Switch prompt refs to github refs

deps-lock.json

@@ -76,14 +76,14 @@
       "hash": "sha256-LaaIDz3x3lGTb/GfarGHaT5MAc3MVCiFWE5Z85qs6z4="
     },
     {
-      "mvn-path": "clj-commons/clj-yaml/1.0.26/clj-yaml-1.0.26.jar",
+      "mvn-path": "clj-commons/clj-yaml/1.0.28/clj-yaml-1.0.28.jar",
       "mvn-repo": "https://repo.clojars.org/",
-      "hash": "sha256-d5eTQ8uSCabGzgSCJiW+B6j4+tWfxTmbpXhzN6fYpTs="
+      "hash": "sha256-QQlD2Qyz5bEzvYqZ4xDrVwZKUE5+642sFrLJi5C3lCw="
     },
     {
-      "mvn-path": "clj-commons/clj-yaml/1.0.26/clj-yaml-1.0.26.pom",
+      "mvn-path": "clj-commons/clj-yaml/1.0.28/clj-yaml-1.0.28.pom",
       "mvn-repo": "https://repo.clojars.org/",
-      "hash": "sha256-89d/qo4tN1NX89YB/1Rr2FnUfeMR/Hpi2q1yHmmwO9I="
+      "hash": "sha256-Un5S4+y7Wzi660rKZ65rPbBNDsImViNc16wLE6FezmU="
     },
     {
       "mvn-path": "com/cognitect/aws/api/0.8.612/api-0.8.612.jar",
@@ -450,16 +450,6 @@
       "mvn-repo": "https://repo.maven.apache.org/maven2/",
       "hash": "sha256-Rm9ue5kB6VKuGtJPDpSUpWgJYfBOQiObWuZOjcr2KXo="
     },
-    {
-      "mvn-path": "markdown-clj/markdown-clj/1.12.1/markdown-clj-1.12.1.jar",
-      "mvn-repo": "https://repo.clojars.org/",
-      "hash": "sha256-WQa0wuwux6osu79nZasLWJIG7AOAk1s2SClZZqhL7L4="
-    },
-    {
-      "mvn-path": "markdown-clj/markdown-clj/1.12.1/markdown-clj-1.12.1.pom",
-      "mvn-repo": "https://repo.clojars.org/",
-      "hash": "sha256-dm4mOOdVHfiYd8zVKAxtkKf9L5AcewxrTbOl39KPPI4="
-    },
     {
       "mvn-path": "net/java/dev/jna/jna-platform/5.15.0/jna-platform-5.15.0.jar",
       "mvn-repo": "https://repo.maven.apache.org/maven2/",
@@ -1401,14 +1391,14 @@
       "hash": "sha256-broJAu/Yma7A2NGaw8vFMSPNQROf4OHSnMXIdKeRud4="
     },
     {
-      "mvn-path": "org/flatland/ordered/1.5.9/ordered-1.5.9.jar",
+      "mvn-path": "org/flatland/ordered/1.15.12/ordered-1.15.12.jar",
       "mvn-repo": "https://repo.clojars.org/",
-      "hash": "sha256-Vl9mSXhByNY9CbtLueqzabUl/HQwGGVRIPowqS+uq5Q="
+      "hash": "sha256-NwD9NzndCSj5SRi3EEWl+IxD+Vr+Z+GjVeuGPi+ZJeA="
     },
     {
-      "mvn-path": "org/flatland/ordered/1.5.9/ordered-1.5.9.pom",
+      "mvn-path": "org/flatland/ordered/1.15.12/ordered-1.15.12.pom",
       "mvn-repo": "https://repo.clojars.org/",
-      "hash": "sha256-wF56Bv8RMDM5GzVmk6rhLh6auW0QbXEXwKqVaO7MYo0="
+      "hash": "sha256-vWUBX+NY7K6ZF+LJ46oSAAY6ika0F5KVjRo1R13H8Oo="
     },
     {
       "mvn-path": "org/fressian/fressian/0.6.6/fressian-0.6.6.jar",
@@ -1546,14 +1536,14 @@
       "hash": "sha256-UGG6hMmFNuWmtM4oD7zssA4zXzsExdSEYpFi/LRiR3g="
     },
     {
-      "mvn-path": "org/yaml/snakeyaml/1.33/snakeyaml-1.33.jar",
+      "mvn-path": "org/yaml/snakeyaml/2.3/snakeyaml-2.3.jar",
       "mvn-repo": "https://repo.maven.apache.org/maven2/",
-      "hash": "sha256-Ef9Fl4jwoteB9WpKhtfmkgLOus0Cc9UmnErp8C8/2PA="
+      "hash": "sha256-Y6dv5mtlI2C9TCwQfm8CWNqn1LtJIAi6jCb80jD/kUY="
     },
     {
-      "mvn-path": "org/yaml/snakeyaml/1.33/snakeyaml-1.33.pom",
+      "mvn-path": "org/yaml/snakeyaml/2.3/snakeyaml-2.3.pom",
       "mvn-repo": "https://repo.maven.apache.org/maven2/",
-      "hash": "sha256-6n1I/UUyGmAz2XzSiBhtSOXpLMDHBm5ziNfEzrSvWVc="
+      "hash": "sha256-D1omWgYzGwBJ41K+MsoyLeGLF/PU27cGNdQNppLjWC8="
     },
     {
       "mvn-path": "pogonos/pogonos/0.2.1/pogonos-0.2.1.jar",

deps.edn

@@ -1,7 +1,7 @@
 {:paths ["src" "dev"]
  :deps {org.clojure/clojure {:mvn/version "1.11.4"}
         org.clojure/core.match {:mvn/version "1.1.0"}
-        markdown-clj/markdown-clj {:mvn/version "1.12.1"}
+        clj-commons/clj-yaml {:mvn/version "1.0.28"}
         pogonos/pogonos {:mvn/version "0.2.1"}
         dev.weavejester/medley {:mvn/version "1.8.0"}
         io.replikativ/hasch {:mvn/version "0.3.94"}

graphs/prompts/journals/2024_09_03.md

@@ -70,9 +70,11 @@
             - find
             - .
             - -name
-  model: llama3.1
-  url:  http://localhost/v1/chat/completions
-  stream: false
+  model: llama3.1                              # override model
+  url:  http://localhost/v1/chat/completions   # override model runner
+  stream: false                                # disable streaming
+  host-dir: /Users/slim/vonwig/altaservice     # override host-dir for testing
+  timeout: 60000                               # timeout for tools used in this prompt
   ---
   
   # Prompt system

prompts/sql/prompt.md

@@ -1,5 +1,6 @@
 ---
 agent: sql
+host-dir: /Users/slim/docker/labs-ai-tools-for-devs/prompts/sql  # override host-dir while testing
 ---
 
 # prompt user

src/docker/main.clj

@@ -178,20 +178,20 @@
                     (user-loop/create-step
                      (fn [state]
                        (let [m (state/construct-initial-state-from-prompts
-                                 (assoc state :opts
-                                        (-> (with-options opts (rest args))
-                                            (assoc :thread-id thread-id))))]
+                                (assoc state :opts
+                                       (-> (with-options opts (rest args))
+                                           (assoc :thread-id thread-id))))]
                          (graph/stream
-                           (if (= (-> m :metadata :agent) "sql")
-                             (graphs.sql/graph state)
-                             (graph/chat-with-tools state))
+                          (if (= (-> m :metadata :agent) "sql")
+                            (graphs.sql/graph state)
+                            (graph/chat-with-tools state))
                           m))))
                     user-loop/state-reducer
                     in
                     {})))
                 opts)))
     (fn []
-      (prompts/get-prompts (with-options opts args)))))
+      (:messages (prompts/get-prompts (with-options opts args))))))
 
 (defn -main [& args]
   (try

src/graph.clj

@@ -30,13 +30,14 @@
   [{:keys [messages functions metadata] {:keys [url model stream level]} :opts}]
   (let [[c h] (openai/chunk-handler)
         request (merge
-                 (dissoc metadata :agent)
+                 (dissoc metadata :agent :host-dir) ; TODO should we just select relevant keys instead of removing bad ones
                  {:messages messages
                   :level level}
                  (when (seq functions) {:tools functions})
+                 ;; overrides from cli opts, NOT from metadata
                  (when url {:url url})
                  (when model {:model model})
-                 ;; stream is a special case where we don't want to overwrite the metadata
+                 ;; stream is a special case where we don't want to allow override of the metadata val if the cli val is set 
                  (when (and stream (nil? (:stream metadata))) {:stream stream}))]
     (try
       (if (seq messages)
@@ -82,8 +83,15 @@
        (into []
              (async/<!
               (->> (tools/make-tool-calls
-                     (or (-> state :opts :level) 0)
-                    (partial tools/function-handler (assoc (:opts state) :functions (:functions state)))
+                    (or (-> state :opts :level) 0)
+                    (partial 
+                      tools/function-handler 
+                      ;; defaults for tool handling are opts, current state functions, and a host-dir override
+                      (merge 
+                        (:opts state) 
+                        (select-keys state [:functions])
+                        ;; note that host-dir, if it exists, is an override here
+                        (select-keys (:metadata state) [:host-dir :timeout])))
                     calls)
                    (async/reduce conj []))))})))
 
@@ -115,11 +123,13 @@
               ((or construct-graph chat-with-tools) state)
               (->
                ((or
+                 ;; the sub-graph might have a function or a vector of state overlays to apply 
                  (and
                   init-state
                   (if (coll? init-state)
                     (apply-functions init-state)
                     init-state))
+                 ;; default is to assume there's a tool call with a function that contains a prompt
                  (comp state/construct-initial-state-from-prompts state/add-prompt-ref)) state)
                (update-in [:opts :level] (fnil inc 0)))))]
         ((or next-state state/add-last-message-as-tool-call) state sub-graph-state)))))

src/graphs/sql.clj

@@ -49,7 +49,7 @@
     (let [x ((graph/apply-functions
               [(state/messages-reset)
                (state/tools-reset)
-               (state/messages-from-prompt "prompts/sql/query-gen.md")
+               (state/messages-from-prompt "github:docker/labs-ai-tools-for-devs?path=prompts/sql/query-gen.md")
                (state/messages-append-all)]) state)
           {:keys [messages _finish-reason]} (async/<! (graph/run-llm x))]
 
@@ -75,8 +75,6 @@
       ;; how many times should we try to correct because correct-query will always end up back here 
       :else "correct-query")))
 
-;; query-gen has a prompt
-;; seed-correct-query-conversation has a prompt
 (defn graph [_]
   (graph/construct-graph
    [[["start"                   graph/start]
@@ -96,7 +94,7 @@
     [["correct-query"           (graph/sub-graph-node
                                  {:init-state
                                   [(state/messages-reset)
-                                   (state/messages-from-prompt "prompts/sql/query-check.md")
+                                   (state/messages-from-prompt "github:docker/labs-ai-tools-for-devs?path=prompts/sql/query-check.md")
                                    (state/messages-take 1)
                                    (state/messages-take-last 1)]
                                   :construct-graph graph/generate-one-tool-call

src/markdown.clj

@@ -1,5 +1,7 @@
 (ns markdown
   (:require
+   [cheshire.core :as json]
+   [clj-yaml.core :as clj-yaml]
    [clojure.core.async :as async]
    [clojure.edn :as edn]
    [clojure.pprint :refer [pprint]]
@@ -61,18 +63,98 @@
    (filter (partial prompt-section? content))
    (map (partial node-content content))))
 
-(defn parse-markdown [content]
+(defn metadata-section? [loc]
+  (= "minus_metadata" (-> loc (zip/node) first)))
+
+(defn remove-markers [s]
+  (and s (when-let [[_ x] (re-find (re-pattern "(?sm).*---(.*)---.*") s)] x)))
+
+(defn extract-metadata [content ast]
+  (try
+    (when-let [loc (->>
+                    (iterate zip/next (zip/seq-zip ast))
+                    (take-while (complement zip/end?))
+                    (some (fn [loc] (when (metadata-section? loc) loc))))]
+      (->
+       (from-range (-> loc zip/node second) content)
+       (remove-markers)
+       (clj-yaml/parse-string)))
+    (catch Throwable _ nil)))
+
+(defn html-comment? [loc]
+  (and
+   (= "html_block" (-> loc (zip/node) first))
+   (= "-->" (-> loc (zip/children) last first))))
+
+(defn extract-first-comment [content ast]
+  (try
+    (when-let [loc (->>
+                     (iterate zip/next (zip/seq-zip ast))
+                     (take-while (complement zip/end?))
+                     (some (fn [loc] (when (html-comment? loc) loc))))]
+      (->
+        (from-range (-> loc zip/node second) content)
+        (remove-markers)
+        (clj-yaml/parse-string)))
+    (catch Throwable ex
+      (println ex)
+      nil)))
+
+(defn parse-new [content query]
   (let [content (str content "\n# END\n\n")
         x (docker/function-call-with-stdin
+           {:image "vonwig/tree-sitter:latest"
+            :content content
+            :command (concat
+                      ["-lang" "markdown"]
+                      ["-query" query])})
+        {s :pty-output} (async/<!! (async/thread
+                                     (Thread/sleep 10)
+                                     (docker/finish-call x)))]
+    (->> s)))
+
+(comment
+  ; TODO - migrate to tree-sitter queries but can we express this with tree-sitter
+  (parse-new (slurp "./tprompt1.md") "(document) @doc")
+  (json/parse-string (parse-new (slurp "./tprompt1.md") "(document (minus_metadata) @doc)"))
+  (json/parse-string (parse-new (slurp "./tprompt1.md") "(document (section (html_block) @html))"))
+  (json/parse-string (parse-new (slurp "./tprompt1.md") "(document (section (atx_heading (atx_h1_marker)))* @top-section)")))
+
+(defn parse-markdown
+  "use the custom sexp representation"
+  [content]
+  (let [x (docker/function-call-with-stdin
            {:image "docker/lsp:treesitter"
             :content content})
         {s :pty-output} (async/<!! (async/thread
                                      (Thread/sleep 10)
                                      (docker/finish-call x)))]
-    (->> s
-         (edn/read-string)
-         (extract-prompts content)
-         (into []))))
+    (->> (edn/read-string s))))
+
+(defn parse-prompts
+  "parse out the h1 prompt sections"
+  [content]
+  (let [content (str content "\n# END\n\n")
+        ast (parse-markdown content)]
+    {:messages
+     (->> ast
+          (extract-prompts content)
+          (into []))
+     :metadata  (or 
+                 (extract-metadata content ast)
+                 (extract-first-comment content ast)) }))
+
+(comment
+  ; inline same line !,[,],(,) in that order after filtering out other irrelevant things
+  ; ^ those are imgages and the content between the [ ] should be put into a separate message
+  ; the first minus_metadata block of the doc 
+  ; the first html_block section that ends with --> 
+  ;   get content and then check of --- --- pre-amble
+  ;   then try to parse the yaml out of that
+  (parse-markdown (slurp "./tprompt2.md"))
+  (parse-prompts (slurp "./tprompt1.md"))
+  (parse-prompts (slurp "./tprompt2.md"))
+  )
 
 (comment
   (string/split content #"\n")