Add files.
This commit is contained in:
parent
191b64e4c4
commit
1656c4cf55
|
|
@ -0,0 +1,8 @@
|
|||
module thread2md
|
||||
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
git.wntrmute.dev/kyle/goutils v1.7.4
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
)
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"git.wntrmute.dev/kyle/goutils/die"
|
||||
"os"
|
||||
"path/filepath"
|
||||
thread2 "thread2md/thread"
|
||||
)
|
||||
|
||||
var janAIPath = filepath.Join(os.Getenv("HOME"), "jan", "threads")
|
||||
|
||||
func main() {
|
||||
var path string
|
||||
flag.StringVar(&path, "path", janAIPath, "path to thread directory")
|
||||
flag.Parse()
|
||||
|
||||
for _, arg := range flag.Args() {
|
||||
thread, err := thread2.LoadThread(filepath.Join(path, arg))
|
||||
die.If(err)
|
||||
|
||||
fmt.Println(thread)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{"id":"01HW8QGD9A4JSSTS6AARX8WFGD","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713985041706,"updated":1713985041706,"object":"thread.message","content":[{"type":"text","text":{"value":"Hello, world.","annotations":[]}}]}
|
||||
{"id":"01HW8QGHZZMVBBW0HHV9MCZPWT","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"Hello, user! How may I help you today?","annotations":[]}}],"status":"ready","created":1713985046527,"updated":1713985046527,"object":"thread.message"}
|
||||
{"id":"01HW8QGYKZ8J5VCNEKJPRAEF2R","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713985059456,"updated":1713985059456,"object":"thread.message","content":[{"type":"text","text":{"value":"What is the air velocity of an unladen swallow?","annotations":[]}}]}
|
||||
{"id":"01HW8QGYMJB2THW8VPNSNJ8QEF","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"The air velocity of an unladen swallow is undefined, as swallows flap their wings to move through the air and do not fly at a constant speed. However, if we were talking about an average flying speed for swallows, it ranges from 15 to 20 miles per hour (24-32 km/h).","annotations":[]}}],"status":"ready","created":1713985059474,"updated":1713985059474,"object":"thread.message"}
|
||||
{"id":"01HW8QHATKM41CRGCRCS8W36VQ","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713985071955,"updated":1713985071955,"object":"thread.message","content":[{"type":"text","text":{"value":"What is your name?","annotations":[]}}]}
|
||||
{"id":"01HW8QHAV4VSYS9MDXJ0WMV22S","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"My name is Writer. I am a text-based application designed to help you with various tasks and provide information. It's not necessary for me to have a specific name, but I can interact with you and assist in different ways","annotations":[]}}],"status":"ready","created":1713985071972,"updated":1713985071972,"object":"thread.message"}
|
||||
{"id":"01HW8QHG9XW2CKSVVTR8AQENBW","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713985077566,"updated":1713985077566,"object":"thread.message","content":[{"type":"text","text":{"value":"What is your quest?","annotations":[]}}]}
|
||||
{"id":"01HW8QHGAGMHHGPNMGFPDGF5CN","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"My quest is to help you and provide information or assistance when needed. I am here to make your interactions more efficient and enjoyable","annotations":[]}}],"status":"ready","created":1713985077584,"updated":1713985077584,"object":"thread.message"}
|
||||
{"id":"01HW8QHQ03PP1KCR5KZ3QG6KJZ","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713985084419,"updated":1713985084419,"object":"thread.message","content":[{"type":"text","text":{"value":"What is your favourite colour?","annotations":[]}}]}
|
||||
{"id":"01HW8QHQ0NFF91EXPZ7YFED0Q6","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"I don't have a favourite color as I am not human, but I can help you with other things. Feel free to ask me anything else!","annotations":[]}}],"status":"ready","created":1713985084437,"updated":1713985084437,"object":"thread.message"}
|
||||
{"id":"01HW8QJ00AHTJHKRTA5H9XEK3V","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713985093642,"updated":1713985093642,"object":"thread.message","content":[{"type":"text","text":{"value":"What is the capital of Assyria?","annotations":[]}}]}
|
||||
{"id":"01HW8QJ00YSM22EPPG6KE1PY3V","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"Assyria was an ancient civilization, and it doesn't have a modern-day capital. The historical Assyrian capitals included Ashur, Nineveh, and Kalhu (modern-day Iraq). However, if you are looking for the capital of Iraq today, it is Baghdad.","annotations":[]}}],"status":"ready","created":1713985093662,"updated":1713985093662,"object":"thread.message"}
|
||||
{"id":"01HW8RK7DZHNQCEY4MKZZ8BXA6","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713986182592,"updated":1713986182592,"object":"thread.message","content":[{"type":"text","text":{"value":"Are you familiar with the application Obsidian?","annotations":[]}}]}
|
||||
{"id":"01HW8RK7EJGEE1HXKSK4CRQXQ2","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"Yes, I am familiar with Obsidian. It is a powerful note-taking and knowledge management application that allows users to create interconnected notes in the form of graphs, making it easier to navigate and explore ideas. I can provide more information or help you with any questions regarding Obsidian if needed","annotations":[]}}],"status":"ready","created":1713986182610,"updated":1713986182610,"object":"thread.message"}
|
||||
{"id":"01HW8RKQJM6NBEXGNPQFW1B32Q","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713986199124,"updated":1713986199124,"object":"thread.message","content":[{"type":"text","text":{"value":"How can I hook an LLM into Obsidian so that I can harness the knowledge from it?","annotations":[]}}]}
|
||||
{"id":"01HW8RKQKHZTTN7H0V11MF06SM","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"To integrate an LLM (Large Language Model) into Obsidian, you can follow these steps:\n1. Install a plugin that supports the integration of LLMs. There are several plugins available for Obsidian that provide this functionality, such as ChatGPT or AI Assistant.\n2. Once the plugin is installed, connect your LLM of choice to the plugin. This may involve configuring API keys and other settings specific to the model you want to use.\n3. With the integration complete, you can now interact with the LLM directly from Obsidian. Simply type a question or prompt into the plugin's interface and receive an answer or response from the model","annotations":[]}}],"status":"ready","created":1713986199153,"updated":1713986199153,"object":"thread.message"}
|
||||
{"id":"01HW8RWH4ZFBBYKQGBWX0967Z5","thread_id":"jan_1713980442","role":"user","status":"ready","created":1713986487455,"updated":1713986487455,"object":"thread.message","content":[{"type":"text","text":{"value":"Are you familiar with jan.ai?","annotations":[]}}]}
|
||||
{"id":"01HW8RWH5HC65MZ82PG0QHH8CX","thread_id":"jan_1713980442","type":"Thread","role":"assistant","content":[{"type":"text","text":{"value":"Yes, I am familiar with jan.ai. It is an app that helps users track their time spent on various activities and provides insights into how they can optimize their day-to-day life. The service offers features like automatic time tracking, productivity analysis, and customizable goals to help users achieve a better work-life balance","annotations":[]}}],"status":"ready","created":1713986487473,"updated":1713986487473,"object":"thread.message"}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"id": "jan_1713980442",
|
||||
"object": "thread",
|
||||
"title": "Greeting in Code",
|
||||
"assistants": [
|
||||
{
|
||||
"assistant_id": "jan",
|
||||
"assistant_name": "Jan",
|
||||
"tools": [
|
||||
{
|
||||
"type": "retrieval",
|
||||
"enabled": false,
|
||||
"settings": {
|
||||
"top_k": 2,
|
||||
"chunk_size": 1024,
|
||||
"chunk_overlap": 64,
|
||||
"retrieval_template": "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n----------------\nCONTEXT: {CONTEXT}\n----------------\nQUESTION: {QUESTION}\n----------------\nHelpful Answer:"
|
||||
}
|
||||
}
|
||||
],
|
||||
"model": {
|
||||
"id": "openhermes-neural-7b",
|
||||
"settings": {
|
||||
"ctx_len": 4096,
|
||||
"prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
|
||||
"llama_model_path": "openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf"
|
||||
},
|
||||
"parameters": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"max_tokens": 4096,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "nitro"
|
||||
},
|
||||
"instructions": ""
|
||||
}
|
||||
],
|
||||
"created": 1713980441779,
|
||||
"updated": 1713986487455,
|
||||
"metadata": {
|
||||
"lastMessage": "Yes, I am familiar with jan.ai. It is an app that helps users track their time spent on various activities and provides insights into how they can optimize their day-to-day life. The service offers features like automatic time tracking, productivity analysis, and customizable goals to help users achieve a better work-life balance"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"id": "jan_1713980442",
|
||||
"object": "thread",
|
||||
"title": "Greeting in Code",
|
||||
"assistants": [
|
||||
{
|
||||
"assistant_id": "jan",
|
||||
"assistant_name": "Jan",
|
||||
"tools": [
|
||||
{
|
||||
"type": "retrieval",
|
||||
"enabled": false,
|
||||
"settings": {
|
||||
"top_k": 2,
|
||||
"chunk_size": 1024,
|
||||
"chunk_overlap": 64,
|
||||
"retrieval_template": "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n----------------\nCONTEXT: {CONTEXT}\n----------------\nQUESTION: {QUESTION}\n----------------\nHelpful Answer:"
|
||||
}
|
||||
}
|
||||
],
|
||||
"model": {
|
||||
"id": "openhermes-neural-7b",
|
||||
"settings": {
|
||||
"ctx_len": 4096,
|
||||
"prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
|
||||
"llama_model_path": "openhermes-2.5-neural-chat-v3-3-slerp.Q4_K_M.gguf"
|
||||
},
|
||||
"parameters": {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.95,
|
||||
"stream": true,
|
||||
"max_tokens": 4096,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0
|
||||
},
|
||||
"engine": "nitro"
|
||||
},
|
||||
"instructions": ""
|
||||
}
|
||||
],
|
||||
"created": 1713980441779,
|
||||
"updated": 1713986487455,
|
||||
"metadata": {
|
||||
"lastMessage": "Yes, I am familiar with jan.ai. It is an app that helps users track their time spent on various activities and provides insights into how they can optimize their day-to-day life. The service offers features like automatic time tracking, productivity analysis, and customizable goals to help users achieve a better work-life balance"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
package thread
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/goutils/die"
|
||||
)
|
||||
|
||||
const timeFormat = "2006-01-02 15:04"
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ThreadID string `json:"thread_id"`
|
||||
Role string `json:"role"`
|
||||
Status string `json:"status"`
|
||||
Created int64 `json:"created"`
|
||||
Updated int64 `json:"updated"`
|
||||
Object string `json:"object"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text struct {
|
||||
Value string `json:"value"`
|
||||
Annotations []any `json:"annotations"`
|
||||
} `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
|
||||
func (m *Message) Date() time.Time {
|
||||
return time.Unix(m.Created/1000, 0)
|
||||
}
|
||||
|
||||
func (m *Message) Header() string {
|
||||
role := m.Role
|
||||
switch role {
|
||||
case "user":
|
||||
role = strings.ToTitle(os.Getenv("USER"))
|
||||
case "assistant":
|
||||
role = "LLM"
|
||||
default:
|
||||
// nothing
|
||||
}
|
||||
return fmt.Sprintf("[%s] **%s**\n", m.Date().Format(timeFormat), m.Role)
|
||||
}
|
||||
|
||||
func (m *Message) Body() string {
|
||||
content := []string{}
|
||||
|
||||
for _, contentItem := range m.Content {
|
||||
// TODO: handle different content types
|
||||
if contentItem.Type != "text" {
|
||||
continue
|
||||
}
|
||||
|
||||
if contentItem.Text.Annotations != nil {
|
||||
log.Print("warning: annotation in content not supported")
|
||||
}
|
||||
|
||||
content = append(content, contentItem.Text.Value)
|
||||
}
|
||||
|
||||
return strings.Join(content, "\n\n")
|
||||
}
|
||||
|
||||
func LoadMessages(path string) ([]*Message, error) {
|
||||
messagesPath := filepath.Join(path, "messages.jsonl")
|
||||
|
||||
file, err := os.Open(messagesPath)
|
||||
die.If(err)
|
||||
defer file.Close()
|
||||
|
||||
jsonParser := json.NewDecoder(file)
|
||||
var messages []*Message
|
||||
|
||||
for err == nil {
|
||||
var message Message
|
||||
err = jsonParser.Decode(&message)
|
||||
if err == nil {
|
||||
messages = append(messages, &message)
|
||||
}
|
||||
}
|
||||
|
||||
if err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
func (m *Message) String() string {
|
||||
return m.Header() + m.Body() + "\n"
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
package thread
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.wntrmute.dev/kyle/goutils/die"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Thread struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Title string `json:"title"`
|
||||
Assistants []struct {
|
||||
AssistantID string `json:"assistant_id"`
|
||||
AssistantName string `json:"assistant_name"`
|
||||
Tools []struct {
|
||||
Type string `json:"type"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Settings struct {
|
||||
TopK int `json:"top_k"`
|
||||
ChunkSize int `json:"chunk_size"`
|
||||
ChunkOverlap int `json:"chunk_overlap"`
|
||||
RetrievalTemplate string `json:"retrieval_template"`
|
||||
} `json:"settings"`
|
||||
} `json:"tools"`
|
||||
Model struct {
|
||||
ID string `json:"id"`
|
||||
Settings struct {
|
||||
CtxLen int `json:"ctx_len"`
|
||||
PromptTemplate string `json:"prompt_template"`
|
||||
LlamaModelPath string `json:"llama_model_path"`
|
||||
} `json:"settings"`
|
||||
Parameters struct {
|
||||
Temperature float64 `json:"temperature"`
|
||||
TopP float64 `json:"top_p"`
|
||||
Stream bool `json:"stream"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
FrequencyPenalty int `json:"frequency_penalty"`
|
||||
PresencePenalty int `json:"presence_penalty"`
|
||||
} `json:"parameters"`
|
||||
Engine string `json:"engine"`
|
||||
} `json:"model"`
|
||||
Instructions string `json:"instructions"`
|
||||
} `json:"assistants"`
|
||||
Created int64 `json:"created"`
|
||||
Updated int64 `json:"updated"`
|
||||
Metadata struct {
|
||||
LastMessage string `json:"lastMessage"`
|
||||
} `json:"metadata"`
|
||||
|
||||
Messages []*Message
|
||||
}
|
||||
|
||||
func LoadThread(path string) (*Thread, error) {
|
||||
realPath := filepath.Join(path, "thread.json")
|
||||
file, err := os.Open(realPath)
|
||||
die.If(err)
|
||||
defer file.Close()
|
||||
|
||||
jsonParser := json.NewDecoder(file)
|
||||
var thread Thread
|
||||
err = jsonParser.Decode(&thread)
|
||||
die.If(err)
|
||||
|
||||
thread.Messages, err = LoadMessages(path)
|
||||
die.If(err)
|
||||
|
||||
return &thread, nil
|
||||
}
|
||||
|
||||
func (t *Thread) Date() time.Time {
|
||||
return time.Unix(int64(t.Created/1000), 0)
|
||||
}
|
||||
|
||||
func (t *Thread) Header() string {
|
||||
return fmt.Sprintf("# %s (%s)\n## %s\n", t.Title,
|
||||
t.Assistants[0].Model.ID, t.Date().Format(timeFormat))
|
||||
}
|
||||
|
||||
func (t *Thread) Body() string {
|
||||
messages := []string{}
|
||||
for _, m := range t.Messages {
|
||||
messages = append(messages, m.String())
|
||||
}
|
||||
|
||||
return strings.Join(messages, "\n")
|
||||
}
|
||||
|
||||
func (t *Thread) String() string {
|
||||
return t.Header() + "\n" + t.Body()
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package thread
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const testThread = "testdata/thread"
|
||||
|
||||
func TestLoadThread(t *testing.T) {
|
||||
thread, err := LoadThread(testThread)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(thread)
|
||||
}
|
||||
Loading…
Reference in New Issue