diff --git a/package.json b/package.json index 7e922a4..aff831b 100644 --- a/package.json +++ b/package.json @@ -130,6 +130,11 @@ "title": "Install slash commands", "category": "DevChat" }, + { + "command": "DevChat.StartLocalService", + "title": "Start local service", + "category": "DevChat" + }, { "command": "DevChat.Chat", "title": "Chat with DevChat", @@ -253,6 +258,10 @@ { "command": "DevChat.InstallCommands", "when": "false" + }, + { + "command": "DevChat.StartLocalService", + "when": "false" } ], "explorer/context": [ diff --git a/src/context/contextRefDefs.ts b/src/context/contextRefDefs.ts deleted file mode 100644 index 4e889f8..0000000 --- a/src/context/contextRefDefs.ts +++ /dev/null @@ -1,340 +0,0 @@ -import * as vscode from 'vscode'; -import * as path from 'path'; - -import { ChatContext } from './contextManager'; - -import { logger } from '../util/logger'; -import { handleCodeSelected } from './contextCodeSelected'; -import DevChat, { ChatOptions } from '../toolwrapper/devchat'; -import { UiUtilWrapper } from '../util/uiUtil'; - - -async function getCurrentSelectText(activeEditor: vscode.TextEditor): Promise { - if (!activeEditor) { - return ""; - } - - const document = activeEditor.document; - const selection = activeEditor.selection; - const selectedText = document.getText(selection); - - return selectedText; -} - -// get full text in activeEditor -async function getFullText(activeEditor: vscode.TextEditor): Promise { - if (!activeEditor) { - return ""; - } - - const document = activeEditor.document; - return document.getText(); -} - -async function getUndefinedSymbols(content: string): Promise { - // run devchat prompt command - const devChat = new DevChat(); - const chatOptions: ChatOptions = {}; - - const onData = (partialResponse) => { }; - const newContent = ` - As a software developer skilled in code analysis, your goal is to examine and understand a provided code snippet. - The code may include various symbols, encompassing both variables and functions. - However, the snippet doesn't include the definitions of some of these symbols. - Now your specific task is to identify and list all such symbols whose definitions are missing but are essential for comprehending the entire code. - This will help in fully grasping the behavior and purpose of the code. Note that the code snippet in question could range from a few lines to a singular symbol. - - Response is json string, don't include any other output except the json object. The json object should be an array of strings, each string is a symbol name: - \`\`\`json - ["f1", "f2"] - \`\`\` - - During this process, you cannot invoke the GPT function. The code snippet is as follows: \n\`\`\`` + content + '```'; ; - - const chatResponse = await devChat.chat(newContent, chatOptions, onData, false); - if (chatResponse && chatResponse.response) { - logger.channel()?.info(chatResponse.response); - } - - // parse data in chatResponse.response - // data format as: - // ```json {data} ``` - // or [data] - // or plain text - // so, parse data between ```json and ``` or directly parse the array, or return an empty array - if (!chatResponse || !chatResponse.response) { - return []; - } - - let responseText = chatResponse.response.trim(); - let symbols: string[]; - - const indexBlock = responseText.indexOf('```'); - if (indexBlock !== -1) { - const indexJsonEnd = responseText.indexOf('```', indexBlock+3); - if (indexJsonEnd !== -1) { - responseText = responseText.substring(indexBlock, indexJsonEnd + 3); - } - } - - if (responseText.startsWith("```") && responseText.endsWith("```")) { - const index = responseText.indexOf('['); - responseText = responseText.substring(index, responseText.length - 3); - try { - symbols = JSON.parse(responseText); - } catch (error) { - return undefined; - } - } else { - try { - symbols = JSON.parse(responseText); - } catch (error) { - return undefined; - } - } - - logger.channel()?.info(`getUndefinedSymbols: ${chatResponse.response}`); - return symbols; -} - -function matchSymbolInline(line: string, symbol: string): number[] { - const escapedSymbol = symbol.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - - // Create a RegExp with the escaped symbol and word boundaries - const regex = new RegExp(`\\b${escapedSymbol}\\b`, 'gu'); - - // Find the match in the selected text - const matches = [...line.matchAll(regex)]; - - // If the symbol is found - if (matches.length === 0) { - return []; - } - - return matches.map((match) => match.index!); -} - -function getMatchedSymbolPositions(selectText: string, symbol: string): object[] { - const lines = selectText.split('\n'); - const positions: object[] = []; - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - const matchedPositions = matchSymbolInline(line, symbol); - for (const pos of matchedPositions) { - positions.push({ - line: i, - character: pos - }); - } - } - return positions; -} - -function isLocationInstance(loc: vscode.Location | vscode.LocationLink): boolean { - try { - return (loc as vscode.Location).uri !== undefined; - } catch (error) { - return false; - } -} - -async function handleCodeSelectedNoFile(fileSelected: string, codeSelected: string, startLine: number) { - const workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath(); - const relativePath = path.relative(workspaceDir!, fileSelected); - - const data = { - path: relativePath, - startLine: startLine, - content: codeSelected - }; - return data; -} - -async function getSymbolDefine(symbolList: string[], activeEditor: vscode.TextEditor, toFile: boolean = true): Promise { - const document = activeEditor!.document; - let selection = activeEditor!.selection; - let selectedText = document!.getText(selection); - if (selectedText === "" && !toFile) { - selectedText = await getFullText(activeEditor); - selection = new vscode.Selection(0, 0, 0, 0); - } - - let contextList: any[] = []; - if (toFile) { - contextList = [await handleCodeSelectedNoFile(document.uri.fsPath, selectedText, selection.start.line)]; - } - let hasVisitedSymbols: Set = new Set(); - let hasPushedSymbols: Set = new Set(); - - // visit each symbol in symbolList, and get it's define - for (const symbol of symbolList) { - logger.channel()?.info(`handle symble: ${symbol} ...`); - // get symbol position in selectedText - // if selectedText is "abc2+abc", symbol is "abc", then symbolPosition is 5 not 0 - // because abc2 is not a symbol - const positions: any[] = getMatchedSymbolPositions(selectedText, symbol); - - for (const pos of positions) { - const symbolPosition = pos.character; - - // if symbol is like a.b.c, then split it to a, b, c - const symbolSplit = symbol.split("."); - let curPosition = 0; - for (const symbolSplitItem of symbolSplit) { - const symbolPositionNew = symbol.indexOf(symbolSplitItem, curPosition) + symbolPosition; - curPosition = symbolPositionNew - symbolPosition + symbolSplitItem.length; - const newPos = new vscode.Position(pos.line + selection.start.line, (pos.line > 0 ? 0 : selection.start.character) + symbolPositionNew); - logger.channel()?.info(`handle sub symble: ${symbolSplitItem} at ${newPos.line}:${newPos.character}`); - - try{ - // call vscode.executeDefinitionProvider - const refLocations = await vscode.commands.executeCommand( - 'vscode.executeDefinitionProvider', - document.uri, - newPos - ); - if (!refLocations) { - logger.channel()?.info(`no def location for ${symbolSplitItem} at ${newPos.line}:${newPos.character}`); - } - - // visit each refLocation, and get it's define - for (const refLocation of refLocations) { - let targetUri: vscode.Uri | undefined = undefined; - let targetRange: vscode.Range | undefined = undefined; - - if (isLocationInstance(refLocation)) { - const locLocation = refLocation as vscode.Location; - targetUri = locLocation.uri; - targetRange = locLocation.range; - } else { - const locLocationLink = refLocation as vscode.LocationLink; - targetUri = locLocationLink.targetUri; - targetRange = locLocationLink.targetRange; - } - if (!targetUri ||!targetRange) { - continue; - } - - logger.channel()?.info(`def location: ${targetUri.fsPath} ${targetRange.start.line}:${targetRange.start.character}-${targetRange.end.line}:${targetRange.end.character}`); - const refLocationString = targetUri.fsPath + "-" + targetRange.start.line + ":" + targetRange.start.character + "-" + targetRange.end.line + ":" + targetRange.end.character; - if (hasVisitedSymbols.has(refLocationString)) { - continue; - } - hasVisitedSymbols.add(refLocationString); - - // get defines in refLocation file - const symbolsT: vscode.DocumentSymbol[] = await vscode.commands.executeCommand( - 'vscode.executeDocumentSymbolProvider', - targetUri - ); - - let targetSymbol: any = undefined; - const visitFun = (symbol: vscode.DocumentSymbol) => { - if (targetRange!.start.isAfterOrEqual(symbol.range.start) && targetRange!.end.isBeforeOrEqual(symbol.range.end)) { - targetSymbol = symbol; - } - - if (targetRange!.start.isAfter(symbol.range.end) || targetRange!.end.isBefore(symbol.range.start)) { - return; - } - - for (const child of symbol.children) { - visitFun(child); - } - }; - for (const symbol of symbolsT) { - visitFun(symbol); - } - - if (targetSymbol !== undefined) { - logger.channel()?.info(`symbol define information: ${targetSymbol.name} at ${targetSymbol.location.uri.fsPath} ${targetSymbol.location.range.start.line}:${targetSymbol.location.range.start.character}-${targetSymbol.location.range.end.line}:${targetSymbol.location.range.end.character}`); - const defLocationString = targetSymbol.location.uri.fsPath + "-" + targetSymbol.location.range.start.line + ":" + targetSymbol.location.range.start.character + "-" + targetSymbol.location.range.end.line + ":" + targetSymbol.location.range.end.character; - if (hasPushedSymbols.has(defLocationString)) { - continue; - } - hasPushedSymbols.add(defLocationString); - - const documentNew = await vscode.workspace.openTextDocument(targetUri); - - if (targetSymbol.kind === vscode.SymbolKind.Variable) { - const renageNew = new vscode.Range(targetSymbol.range.start.line, 0, targetSymbol.range.end.line, 10000); - if (toFile) { - contextList.push(await handleCodeSelected(targetUri.fsPath, documentNew.getText(renageNew), targetSymbol.range.start.line)); - } else { - contextList.push(await handleCodeSelectedNoFile(targetUri.fsPath, documentNew.getText(renageNew), targetSymbol.range.start.line)); - } - } else { - if (toFile) { - contextList.push(await handleCodeSelected(targetUri.fsPath, documentNew.getText(targetSymbol.range), targetSymbol.range.start.line)); - } else { - contextList.push(await handleCodeSelectedNoFile(targetUri.fsPath, documentNew.getText(targetSymbol.range), targetSymbol.range.start.line)); - } - } - } - } - } catch (error) { - logger.channel()?.error(`getSymbolDefine error: ${error}`); - } - } - } - } - return contextList; -} - -export async function getSymbolDefines() { - const activeEditor = vscode.window.activeTextEditor; - - if (!activeEditor) { - logger.channel()?.error('No code selected!'); - logger.channel()?.show(); - return []; - } - - let selectedText = await getCurrentSelectText(activeEditor); - if (selectedText === "") { - selectedText = await getFullText(activeEditor); - } - if (selectedText === "") { - logger.channel()?.error(`No code selected! Current selected editor is: ${activeEditor.document.uri.fsPath}}`); - logger.channel()?.show(); - return []; - } - const symbolList = await getUndefinedSymbols(selectedText); - if (symbolList === undefined) { - logger.channel()?.error('Failed to get symbol list!'); - logger.channel()?.show(); - return []; - } - const contextList = await getSymbolDefine(symbolList, activeEditor, false); - return contextList; -} - -export const refDefsContext: ChatContext = { - name: 'symbol definitions', - description: 'find related definitions of classes, functions, etc. in selected code', - handler: async () => { - const activeEditor = vscode.window.activeTextEditor; - - if (!activeEditor) { - logger.channel()?.error('No code selected!'); - logger.channel()?.show(); - return []; - } - - const selectedText = await getCurrentSelectText(activeEditor); - if (selectedText === "") { - logger.channel()?.error(`No code selected! Current selected editor is: ${activeEditor.document.uri.fsPath}}`); - logger.channel()?.show(); - return []; - } - const symbolList = await getUndefinedSymbols(selectedText); - if (symbolList === undefined) { - logger.channel()?.error('Failed to get symbol list!'); - logger.channel()?.show(); - return []; - } - const contextList = await getSymbolDefine(symbolList, activeEditor); - - return contextList; - }, -}; diff --git a/src/contributes/commands.ts b/src/contributes/commands.ts index e172e81..23969a9 100644 --- a/src/contributes/commands.ts +++ b/src/contributes/commands.ts @@ -8,11 +8,13 @@ import { FilePairManager } from "../util/diffFilePairs"; import { UiUtilWrapper } from "../util/uiUtil"; import { sendCommandListByDevChatRun } from '../handler/workflowCommandHandler'; -import DevChat from "../toolwrapper/devchat"; +import { DevChatClient } from "../toolwrapper/devchatClient"; import { chatWithDevChat } from '../handler/chatHandler'; import { focusDevChatInput } from '../handler/focusHandler'; import { DevChatConfig } from '../util/config'; import { MessageHandler } from "../handler/messageHandler"; +import { startLocalService } from '../util/localService'; +import { logger } from "../util/logger"; const readdir = util.promisify(fs.readdir); const mkdir = util.promisify(fs.mkdir); @@ -196,7 +198,7 @@ export function registerInstallCommandsCommand( "workflowsCommands" ); // Adjust this path as needed - const devchat = new DevChat(); + const dcClient = new DevChatClient(); if (!fs.existsSync(sysDirPath)) { await copyDirectory(pluginDirPath, sysDirPath); @@ -204,15 +206,15 @@ export function registerInstallCommandsCommand( // Check if ~/.chat/scripts directory exists if (!fs.existsSync(sysDirPath)) { - // Directory does not exist, wait for updateSysCommand to finish - await devchat.updateSysCommand(); + // Directory does not exist, wait for updateWorkflows to finish + await dcClient.updateWorkflows(); sendCommandListByDevChatRun(); } else { // Directory exists, execute sendCommandListByDevChatRun immediately await sendCommandListByDevChatRun(); - // Then asynchronously execute updateSysCommand - await devchat.updateSysCommand(); + // Then asynchronously execute updateWorkflows + await dcClient.updateWorkflows(); await sendCommandListByDevChatRun(); } } @@ -222,6 +224,28 @@ export function registerInstallCommandsCommand( } +export function registerStartLocalServiceCommand( + context: vscode.ExtensionContext +) { + let disposable = vscode.commands.registerCommand( + "DevChat.StartLocalService", + async () => { + try { + const workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath() ?? ''; + logger.channel()?.debug(`extensionPath: ${context.extensionPath}`); + logger.channel()?.debug(`workspacePath: ${workspaceDir}`); + const port = await startLocalService(context.extensionPath, workspaceDir); + logger.channel()?.debug(`Local service started on port ${port}`); + } catch (error) { + logger.channel()?.error('Failed to start local service:', error); + } + } + ); + + context.subscriptions.push(disposable); +} + + export function registerDevChatChatCommand(context: vscode.ExtensionContext) { let disposable = vscode.commands.registerCommand( "DevChat.Chat", diff --git a/src/extension.ts b/src/extension.ts index 50f486e..bcddbf5 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -16,6 +16,7 @@ import { registerFixCommand, registerExplainCommand, registerQuickFixCommand, + registerStartLocalServiceCommand } from './contributes/commands'; import { regLanguageContext } from './contributes/context'; import { regDevChatView } from './contributes/views'; @@ -33,6 +34,7 @@ import { DevChatConfig } from './util/config'; import { InlineCompletionProvider, registerCodeCompleteCallbackCommand } from "./contributes/codecomplete/codecomplete"; import { indexDir } from "./contributes/codecomplete/astIndex"; import registerQuickFixProvider from "./contributes/quickFixProvider"; +import { stopLocalService } from './util/localService'; async function migrateConfig() { @@ -152,6 +154,7 @@ async function activate(context: vscode.ExtensionContext) { registerStatusBarItemClickCommand(context); registerInstallCommandsCommand(context); + registerStartLocalServiceCommand(context); createStatusBarItem(context); @@ -178,6 +181,13 @@ async function activate(context: vscode.ExtensionContext) { async function deactivate() { // stop devchat await stopDevChatBase({}); + + try { + await stopLocalService(); + logger.channel()?.info('Local service stopped successfully'); + } catch (error) { + logger.channel()?.error('Error stopping local service:', error); + } } exports.activate = activate; exports.deactivate = deactivate; diff --git a/src/handler/handlerRegister.ts b/src/handler/handlerRegister.ts index cdcb1d9..12545cf 100644 --- a/src/handler/handlerRegister.ts +++ b/src/handler/handlerRegister.ts @@ -3,7 +3,7 @@ import { insertCodeBlockToFile } from './codeBlockHandler'; import { replaceCodeBlockToFile } from './codeBlockHandler'; import { doCommit } from './commitHandler'; import { getHistoryMessages } from './historyMessagesHandler'; -import { getWorkflowCommandList } from './workflowCommandHandler'; +import { handleRegCommandList } from './workflowCommandHandler'; import { sendMessage, stopDevChat, regeneration, deleteChatMessage, userInput } from './sendMessage'; import { applyCodeWithDiff } from './diffHandler'; import { addConext } from './contextHandler'; @@ -36,7 +36,7 @@ messageHandler.registerHandler('doCommit', doCommit); messageHandler.registerHandler('historyMessages', getHistoryMessages); // Register the command list // Response: { command: 'regCommandList', result: } -messageHandler.registerHandler('regCommandList', getWorkflowCommandList); +messageHandler.registerHandler('regCommandList', handleRegCommandList); // Send a message, send the message entered by the user to AI // Response: // { command: 'receiveMessagePartial', text: , user: , date: } diff --git a/src/handler/historyMessagesBase.ts b/src/handler/historyMessagesBase.ts index dd4d0ce..eb2161b 100644 --- a/src/handler/historyMessagesBase.ts +++ b/src/handler/historyMessagesBase.ts @@ -1,4 +1,17 @@ -import DevChat, { LogEntry, LogOptions } from '../toolwrapper/devchat'; +import { DevChatClient, ShortLog } from '../toolwrapper/devchatClient'; + +export interface LogEntry { + hash: string; + parent: string | null; + user: string; + date: string; + request: string; + response: string; + context: Array<{ + content: string; + role: string; + }>; +} export interface LoadHistoryMessages { command: string; @@ -41,18 +54,28 @@ OPENAI_API_KEY is missing from your environment or settings. Kindly input your O } as LogEntry; } -export async function loadTopicHistoryLogs(topicId: string | undefined): Promise | undefined> { +async function loadTopicHistoryLogs(topicId: string | undefined): Promise | undefined> { if (!topicId) { return undefined; } - - const devChat = new DevChat(); - const logOptions: LogOptions = { - skip: 0, - maxCount: 10000, - topic: topicId - }; - const logEntries = await devChat.log(logOptions); + + const dcClient = new DevChatClient(); + const shortLogs: ShortLog[] = await dcClient.getTopicLogs(topicId, 10000, 0); + + const logEntries: Array = []; + for (const shortLog of shortLogs) { + const logE: LogEntry = { + hash: shortLog.hash, + parent: shortLog.parent, + user: shortLog.user, + date: shortLog.date, + request: shortLog.request, + response: shortLog.responses[0], + context: shortLog.context, + }; + + logEntries.push(logE); + } return logEntries; } diff --git a/src/handler/sendMessageBase.ts b/src/handler/sendMessageBase.ts index e8d5cd2..7842d33 100644 --- a/src/handler/sendMessageBase.ts +++ b/src/handler/sendMessageBase.ts @@ -1,6 +1,8 @@ -import DevChat, { ChatOptions, ChatResponse } from '../toolwrapper/devchat'; import { logger } from '../util/logger'; import { assertValue } from '../util/check'; +import { DevChatClient, ChatRequest, ChatResponse, buildRoleContextsFromFiles, LogData } from '../toolwrapper/devchatClient'; +import { DevChatCLI } from '../toolwrapper/devchatCLI'; +import { ApiKeyManager } from '../util/apiKey'; /** @@ -99,6 +101,15 @@ export function parseMessage(message: string): { context: string[]; instruction: return { context: contextPaths, instruction: instructionPaths, reference: referencePaths, text }; } + +// TODO: to be removed later +interface ChatOptions { + parent?: string; + reference?: string[]; + header?: string[]; + context?: string[]; +} + /** * Parses a message and sets the chat options based on the parsed message. * @@ -130,7 +141,9 @@ export function processChatResponse(chatResponse: ChatResponse) : string { } -const devChat = new DevChat(); +// const devChat = new DevChat(); +const dcClient = new DevChatClient(); +const dcCLI = new DevChatCLI(); /** * Sends a message to the DevChat and handles the response. @@ -148,24 +161,89 @@ export async function sendMessageBase(message: any, handlePartialData: (data: { const [parsedMessage, chatOptions] = await parseMessageAndSetOptions(message); logger.channel()?.trace(`parent hash: ${chatOptions.parent}`); - // call devchat chat - const chatResponse = await devChat.chat( - parsedMessage.text, - chatOptions, - (partialResponse: ChatResponse) => { - const partialDataText = partialResponse.response.replace(/```\ncommitmsg/g, "```commitmsg"); - handlePartialData({ command: 'receiveMessagePartial', text: partialDataText!, user: partialResponse.user, date: partialResponse.date }); - }); + + // send chat message to devchat service + const llmModelData = await ApiKeyManager.llmModel(); + assertValue(!llmModelData || !llmModelData.model, 'You must select a LLM model to use for conversations'); + const chatReq: ChatRequest = { + content: parsedMessage.text, + model_name: llmModelData.model, + api_key: llmModelData.api_key, + api_base: llmModelData.api_base, + parent: chatOptions.parent, + context: chatOptions.context, + }; + let chatResponse = await dcClient.message( + chatReq, + (partialRes: ChatResponse) => { + const text = partialRes.response; + handlePartialData({ + command: "receiveMessagePartial", + text: text!, + user: partialRes.user, + date: partialRes.date, + }); + } + ); + + let workflowRes: ChatResponse | undefined = undefined; + if (chatResponse.finish_reason === "should_run_workflow") { + // invoke workflow via cli + workflowRes = await dcCLI.runWorkflow( + parsedMessage.text, + chatOptions, + (partialResponse: ChatResponse) => { + const partialDataText = partialResponse.response; + handlePartialData({ + command: "receiveMessagePartial", + text: partialDataText!, + user: partialResponse.user, + date: partialResponse.date, + }); + } + ); + } + + const finalResponse = workflowRes || chatResponse; + + // insert log + const roleContexts = await buildRoleContextsFromFiles(chatOptions.context); + const messages = [ + { + role: "user", + content: chatReq.content, + }, + { + role: "assistant", + content: finalResponse.response, + }, + ...roleContexts + ]; + + const logData: LogData = { + model: llmModelData.model, + messages: messages, + parent: chatOptions.parent, + timestamp: Math.floor(Date.now()/1000), + // TODO: 1 or real value? + request_tokens: 1, + response_tokens: 1, + }; + const logRes = await dcClient.insertLog(logData); + + if (logRes.hash) { + finalResponse["prompt-hash"] = logRes.hash; + } assertValue(UserStopHandler.isUserInteractionStopped(), "User Stopped"); return { command: 'receiveMessage', - text: processChatResponse(chatResponse), - hash: chatResponse['prompt-hash'], - user: chatResponse.user, - date: chatResponse.date, - isError: chatResponse.isError + text: processChatResponse(finalResponse), + hash: finalResponse['prompt-hash'], + user: finalResponse.user, + date: finalResponse.date, + isError: finalResponse.isError }; } catch (error: any) { logger.channel()?.error(`Error occurred while sending response: ${error.message}`); @@ -184,7 +262,8 @@ export async function sendMessageBase(message: any, handlePartialData: (data: { export async function stopDevChatBase(message: any): Promise { logger.channel()?.info(`Stopping devchat`); UserStopHandler.stopUserInteraction(); - devChat.stop(); + dcClient.stopAllRequest(); + dcCLI.stop(); } /** @@ -200,9 +279,10 @@ export async function deleteChatMessageBase(message:{'hash': string}): Promise} - A Promise that resolves when the message has been sent. */ export async function sendTextToDevChat(text: string): Promise { - return devChat.input(text); + dcCLI.input(text); + return; } \ No newline at end of file diff --git a/src/handler/topicHandler.ts b/src/handler/topicHandler.ts index fe55221..df2c7ed 100644 --- a/src/handler/topicHandler.ts +++ b/src/handler/topicHandler.ts @@ -1,67 +1,62 @@ -import * as vscode from 'vscode'; -import * as os from 'os'; -import * as path from 'path'; -import * as fs from 'fs'; -import { regInMessage, regOutMessage } from '../util/reg_messages'; -import { MessageHandler } from './messageHandler'; -import DevChat, { TopicEntry } from '../toolwrapper/devchat'; -import { UiUtilWrapper } from '../util/uiUtil'; +import * as vscode from "vscode"; +import { regInMessage, regOutMessage } from "../util/reg_messages"; +import { MessageHandler } from "./messageHandler"; +import { DevChatClient } from "../toolwrapper/devchatClient"; +import { LogEntry } from "./historyMessagesBase"; +const dcClient = new DevChatClient(); -function isDeleteTopic(topicId: string) { - let workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath(); - if (!workspaceDir) { - workspaceDir = os.homedir(); - } - const deletedTopicsPath = path.join(workspaceDir!, '.chat', '.deletedTopics'); - - if (!fs.existsSync(deletedTopicsPath)) { - return false; - } - - const deletedTopics = fs.readFileSync(deletedTopicsPath, 'utf-8').split('\n'); - // check whether topicId in deletedTopics - return deletedTopics.includes(topicId); +export interface TopicEntry { + // eslint-disable-next-line @typescript-eslint/naming-convention + root_prompt: LogEntry; + // eslint-disable-next-line @typescript-eslint/naming-convention + latest_time: number; + hidden: boolean; + title: string | null; } // 注册获取当前topic列表的命令 -regInMessage({ command: 'getTopics' }); -regOutMessage({ command: 'getTopics', topics: [] }); -export async function getTopics(message: any, panel: vscode.WebviewPanel | vscode.WebviewView): Promise { - const devChat = new DevChat(); - const topicEntriesAll: TopicEntry[] = await devChat.topics(); +regInMessage({ command: "getTopics" }); +regOutMessage({ command: "getTopics", topics: [] }); +export async function getTopics( + message: any, + panel: vscode.WebviewPanel | vscode.WebviewView +): Promise { + const topics = await dcClient.getTopics(100, 0); + const entries: TopicEntry[] = []; - let topicEntries: TopicEntry[] = []; - for (const topicEntry of topicEntriesAll) { - if (isDeleteTopic(topicEntry.root_prompt.hash)) { - continue; - } - // append topicEntry to topicEntries - topicEntries.push(topicEntry); + for (const topic of topics) { + const rootLog: LogEntry = { + hash: topic.root_prompt_hash, + parent: topic.root_prompt_parent, + user: topic.root_prompt_user, + date: topic.root_prompt_date, + request: topic.root_prompt_request, + response: topic.root_prompt_response, + context: [], + }; + const e: TopicEntry = { + root_prompt: rootLog, + latest_time: topic.latest_time, + hidden: topic.hidden, + title: topic.title, + }; + entries.push(e); } - MessageHandler.sendMessage(panel, { command: 'getTopics', topicEntries }); + MessageHandler.sendMessage(panel, { + command: "getTopics", + topicEntries: entries, + }); } // 注册删除topic的命令 -regInMessage({ command: 'deleteTopic', topicId: '' }); -export async function deleteTopic(message: any, panel: vscode.WebviewPanel | vscode.WebviewView): Promise { +regInMessage({ command: "deleteTopic", topicId: "" }); +export async function deleteTopic( + message: any, + panel: vscode.WebviewPanel | vscode.WebviewView +): Promise { const topicId = message.topicId; - let workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath(); - if (!workspaceDir) { - workspaceDir = os.homedir(); - } - const deletedTopicsPath = path.join(workspaceDir!, '.chat', '.deletedTopics'); - - // read ${WORKSPACE_ROOT}/.chat/.deletedTopics as String[] - // add topicId to String[] - // write String[] to ${WORKSPACE_ROOT}/.chat/.deletedTopics - let deletedTopics: string[] = []; - - if (fs.existsSync(deletedTopicsPath)) { - deletedTopics = fs.readFileSync(deletedTopicsPath, 'utf-8').split('\n'); - } - deletedTopics.push(topicId); - fs.writeFileSync(deletedTopicsPath, deletedTopics.join('\n')); + await dcClient.deleteTopic(topicId); } diff --git a/src/handler/workflowCommandHandler.ts b/src/handler/workflowCommandHandler.ts index 4c99330..c886116 100644 --- a/src/handler/workflowCommandHandler.ts +++ b/src/handler/workflowCommandHandler.ts @@ -1,24 +1,60 @@ -import * as vscode from 'vscode'; -import { MessageHandler } from './messageHandler'; -import { regInMessage, regOutMessage } from '../util/reg_messages'; -import { ApiKeyManager } from '../util/apiKey'; -import DevChat from '../toolwrapper/devchat'; +import * as vscode from "vscode"; +import { MessageHandler } from "./messageHandler"; +import { regInMessage, regOutMessage } from "../util/reg_messages"; +import { DevChatClient } from "../toolwrapper/devchatClient"; +import { logger } from "../util/logger"; +let existPannel: vscode.WebviewPanel | vscode.WebviewView | undefined = + undefined; -let existPannel: vscode.WebviewPanel|vscode.WebviewView|undefined = undefined; +regInMessage({ command: "regCommandList" }); +regOutMessage({ + command: "regCommandList", + result: [{ name: "", pattern: "", description: "" }], +}); +export async function handleRegCommandList( + message: any, + panel: vscode.WebviewPanel | vscode.WebviewView +): Promise { + existPannel = panel; + if (process.env.DC_LOCALSERVICE_PORT) { + await getWorkflowCommandList(message, existPannel!); + } +} -regInMessage({command: 'regCommandList'}); -regOutMessage({command: 'regCommandList', result: [{name: '', pattern: '', description: ''}]}); -export async function getWorkflowCommandList(message: any, panel: vscode.WebviewPanel|vscode.WebviewView): Promise { - existPannel = panel; +export async function getWorkflowCommandList( + message: any, + panel: vscode.WebviewPanel | vscode.WebviewView +): Promise { + const dcClient = new DevChatClient(); - const commandList = await new DevChat().commands(); - MessageHandler.sendMessage(panel, { command: 'regCommandList', result: commandList }); - return; + // All workflows registered in DevChat + const workflows = await dcClient.getWorkflowList(); + + // Get recommends from config + const workflowsConfig = await dcClient.getWorkflowConfig(); + const recommends = workflowsConfig.recommend?.workflows || []; + + // Filter active workflows and add recommend info + const commandList = workflows + .filter((workflow) => workflow.active) + .map((workflow: any) => ({ + ...workflow, + recommend: recommends.indexOf(workflow.name), + })); + + if (commandList.length > 0) { + MessageHandler.sendMessage(panel, { + command: "regCommandList", + result: commandList, + }); + } + + return; } export async function sendCommandListByDevChatRun() { - if (existPannel) { - await getWorkflowCommandList({}, existPannel!); - } + if (existPannel) { + await getWorkflowCommandList({}, existPannel!); + } } diff --git a/src/ide_services/endpoints/unofficial.ts b/src/ide_services/endpoints/unofficial.ts index 3bf29a9..750b89e 100644 --- a/src/ide_services/endpoints/unofficial.ts +++ b/src/ide_services/endpoints/unofficial.ts @@ -1,6 +1,4 @@ -import * as vscode from "vscode"; import { applyCodeWithDiff, applyEditCodeWithDiff } from "../../handler/diffHandler"; -import { getSymbolDefines } from "../../context/contextRefDefs"; export namespace UnofficialEndpoints { @@ -13,16 +11,6 @@ export namespace UnofficialEndpoints { return true; } - export async function getSymbolDefinesInSelectedCode() { - // find needed symbol defines in current editor document - // return value is a list of symbol defines - // each define has three fileds: - // path: file path contain that symbol define - // startLine: start line in that file - // content: source code for symbol define - return getSymbolDefines(); - } - export async function runCode(code: string) { // run code // delcare use vscode diff --git a/src/ide_services/services.ts b/src/ide_services/services.ts index d23f28a..f70f81f 100644 --- a/src/ide_services/services.ts +++ b/src/ide_services/services.ts @@ -81,10 +81,6 @@ const functionRegistry: any = { /** * Unofficial endpoints */ - "/get_symbol_defines_in_selected_code": { - keys: [], - handler: UnofficialEndpoints.getSymbolDefinesInSelectedCode, - }, "/run_code": { keys: ["code"], handler: UnofficialEndpoints.runCode, diff --git a/src/panel/statusBarView.ts b/src/panel/statusBarView.ts index 9f2ada8..c44d84e 100644 --- a/src/panel/statusBarView.ts +++ b/src/panel/statusBarView.ts @@ -51,6 +51,7 @@ export function createStatusBarItem(context: vscode.ExtensionContext): vscode.St // install devchat workflow commands if (!hasInstallCommands) { hasInstallCommands = true; + await vscode.commands.executeCommand('DevChat.StartLocalService'); await vscode.commands.executeCommand('DevChat.InstallCommands'); // vscode.commands.executeCommand('DevChat.InstallCommandPython'); } diff --git a/src/toolwrapper/devchat.ts b/src/toolwrapper/devchat.ts deleted file mode 100644 index 1e643f9..0000000 --- a/src/toolwrapper/devchat.ts +++ /dev/null @@ -1,545 +0,0 @@ -import * as dotenv from 'dotenv'; -import * as path from 'path'; -import * as fs from 'fs'; -import * as os from 'os'; - -import { logger } from '../util/logger'; -import { CommandRun } from "../util/commonUtil"; -import { UiUtilWrapper } from '../util/uiUtil'; -import { ApiKeyManager } from '../util/apiKey'; -import { assertValue } from '../util/check'; -import { getFileContent } from '../util/commonUtil'; -import * as toml from '@iarna/toml'; -import { DevChatConfig } from '../util/config'; - -import { getMicromambaUrl } from '../util/python_installer/conda_url'; - -const readFileAsync = fs.promises.readFile; - -const envPath = path.join(__dirname, '..', '.env'); -dotenv.config({ path: envPath }); - -export interface ChatOptions { - parent?: string; - reference?: string[]; - header?: string[]; - context?: string[]; -} - -export interface LogOptions { - skip?: number; - maxCount?: number; - topic?: string; -} - -export interface LogEntry { - hash: string; - parent: string; - user: string; - date: string; - request: string; - response: string; - context: Array<{ - content: string; - role: string; - }>; -} - -export interface CommandEntry { - name: string; - description: string; - path: string; - // default value is -1, which means not recommended - recommend: number; -} - -export interface TopicEntry { - // eslint-disable-next-line @typescript-eslint/naming-convention - root_prompt: LogEntry; - // eslint-disable-next-line @typescript-eslint/naming-convention - latest_time: number; - hidden: boolean; - title: string | null; -} - -export interface ChatResponse { - // eslint-disable-next-line @typescript-eslint/naming-convention - "prompt-hash": string; - user: string; - date: string; - response: string; - // eslint-disable-next-line @typescript-eslint/naming-convention - finish_reason: string; - isError: boolean; -} - - -class DevChat { - private commandRun: CommandRun; - - constructor() { - this.commandRun = new CommandRun(); - } - - private async loadContextsFromFiles(contexts: string[] | undefined): Promise { - if (!contexts) { - return []; - } - - const loadedContexts: string[] = []; - for (const context of contexts) { - const contextContent = await getFileContent(context); - if (!contextContent) { - continue; - } - - loadedContexts.push(contextContent); - } - return loadedContexts; - } - - private async buildArgs(options: ChatOptions): Promise { - let args = ["-m", "devchat", "route"]; - - if (options.reference) { - for (const reference of options.reference) { - args.push("-r", reference); - } - } - if (options.header) { - for (const header of options.header) { - args.push("-i", header); - } - } - if (options.context) { - for (const context of options.context) { - args.push("-c", context); - } - } - - if (options.parent) { - args.push("-p", options.parent); - } - - const llmModelData = await ApiKeyManager.llmModel(); - assertValue(!llmModelData || !llmModelData.model, 'You must select a LLM model to use for conversations'); - args.push("-m", llmModelData.model); - - const functionCalling = DevChatConfig.getInstance().get('enable_function_calling'); - if (functionCalling) { - args.push("-a"); - } - - return args; - } - - private buildLogArgs(options: LogOptions): string[] { - let args = ["-m", "devchat", "log"]; - - if (options.skip) { - args.push('--skip', `${options.skip}`); - } - if (options.maxCount) { - args.push('--max-count', `${options.maxCount}`); - } else { - const maxLogCount = DevChatConfig.getInstance().get('max_log_count'); - args.push('--max-count', `${maxLogCount}`); - } - - if (options.topic) { - args.push('--topic', `${options.topic}`); - } - - return args; - } - - private parseOutData(stdout: string, isPartial: boolean): ChatResponse { - const responseLines = stdout.trim().split("\n"); - - if (responseLines.length < 2) { - return this.createChatResponse("", "", "", "", !isPartial); - } - - const [userLine, remainingLines1] = this.extractLine(responseLines, "User: "); - const user = this.parseLine(userLine, /User: (.+)/); - - const [dateLine, remainingLines2] = this.extractLine(remainingLines1, "Date: "); - const date = this.parseLine(dateLine, /Date: (.+)/); - - const [promptHashLine, remainingLines3] = this.extractLine(remainingLines2, "prompt"); - const [finishReasonLine, remainingLines4] = this.extractLine(remainingLines3, "finish_reason:"); - - if (!promptHashLine) { - return this.createChatResponse("", user, date, remainingLines4.join("\n"), !isPartial); - } - - const finishReason = finishReasonLine.split(" ")[1]; - const promptHash = promptHashLine.split(" ")[1]; - const response = remainingLines4.join("\n"); - - return this.createChatResponse(promptHash, user, date, response, false, finishReason); - } - - private extractLine(lines: string[], startWith: string): [string, string[]] { - const index = lines.findIndex(line => line.startsWith(startWith)); - const extractedLine = index !== -1 ? lines.splice(index, 1)[0] : ""; - return [extractedLine, lines]; - } - - private parseLine(line: string, regex: RegExp): string { - return (line.match(regex)?.[1]) ?? ""; - } - - private createChatResponse(promptHash: string, user: string, date: string, response: string, isError: boolean, finishReason = ""): ChatResponse { - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - "prompt-hash": promptHash, - user, - date, - response, - // eslint-disable-next-line @typescript-eslint/naming-convention - finish_reason: finishReason, - isError, - }; - } - - private async runCommand(args: string[]): Promise<{code: number | null, stdout: string, stderr: string}> { - // build env variables for command - const envs = { - ...process.env, - // eslint-disable-next-line @typescript-eslint/naming-convention - "PYTHONUTF8":1, - // eslint-disable-next-line @typescript-eslint/naming-convention - "PYTHONPATH": UiUtilWrapper.extensionPath() + "/tools/site-packages", - // eslint-disable-next-line @typescript-eslint/naming-convention - "DEVCHAT_PROXY": DevChatConfig.getInstance().get('DEVCHAT_PROXY') || "", - "MAMBA_BIN_PATH": getMicromambaUrl(), - }; - - const pythonApp = DevChatConfig.getInstance().get('python_for_chat') || "python3"; - - // run command - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync( - pythonApp, - args, - { - maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB - cwd: UiUtilWrapper.workspaceFoldersFirstPath(), - env: envs - }, - undefined, undefined, undefined, undefined - ); - - return {code, stdout, stderr}; - } - - public input(data: string) { - this.commandRun?.write(data + "\n"); - } - - public stop() { - this.commandRun.stop(); - } - - async chat(content: string, options: ChatOptions = {}, onData: (data: ChatResponse) => void, saveToLog: boolean = true): Promise { - try { - // build args for devchat prompt command - const args = await this.buildArgs(options); - args.push("--"); - args.push(content); - - // build env variables for prompt command - const llmModelData = await ApiKeyManager.llmModel(); - assertValue(!llmModelData, "No valid llm model selected"); - const envs = { - ...process.env, - // eslint-disable-next-line @typescript-eslint/naming-convention - "PYTHONUTF8": 1, - // eslint-disable-next-line @typescript-eslint/naming-convention - "command_python": DevChatConfig.getInstance().get('python_for_commands') || "", - // eslint-disable-next-line @typescript-eslint/naming-convention - "PYTHONPATH": UiUtilWrapper.extensionPath() + "/tools/site-packages", - // eslint-disable-next-line @typescript-eslint/naming-convention - "OPENAI_API_KEY": llmModelData.api_key.trim(), - "DEVCHAT_UNIT_TESTS_USE_USER_MODEL": 1, - // eslint-disable-next-line @typescript-eslint/naming-convention - ...llmModelData.api_base? { "OPENAI_API_BASE": llmModelData.api_base, "OPENAI_BASE_URL": llmModelData.api_base } : {}, - "DEVCHAT_PROXY": DevChatConfig.getInstance().get('DEVCHAT_PROXY') || "", - "MAMBA_BIN_PATH": getMicromambaUrl(), - }; - - // build process options - const spawnAsyncOptions = { - maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB - cwd: UiUtilWrapper.workspaceFoldersFirstPath(), - env: envs - }; - - logger.channel()?.info(`api_key: ${llmModelData.api_key.replace(/^(.{4})(.*)(.{4})$/, (_, first, middle, last) => first + middle.replace(/./g, '*') + last)}`); - logger.channel()?.info(`api_base: ${llmModelData.api_base}`); - - // run command - // handle stdout as steam mode - let receviedStdout = ""; - const onStdoutPartial = (stdout: string) => { - receviedStdout += stdout; - const data = this.parseOutData(receviedStdout, true); - onData(data); - }; - // run command - const pythonApp = DevChatConfig.getInstance().get('python_for_chat') || "python3"; - logger.channel()?.info(`Running devchat:${pythonApp} ${args.join(" ")}`); - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, args, spawnAsyncOptions, onStdoutPartial, undefined, undefined, undefined); - // handle result - assertValue(code !== 0, stderr || "Command exited with error code"); - const responseData = this.parseOutData(stdout, false); - let promptHash = ""; - if (saveToLog) { - const logs = await this.logInsert(options.context, content, responseData.response, options.parent); - assertValue(!logs || !logs.length, "Failed to insert devchat log"); - promptHash = logs[0]["hash"]; - } - // return result - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - "prompt-hash": promptHash, - user: "", - date: "", - response: responseData.response, - // eslint-disable-next-line @typescript-eslint/naming-convention - finish_reason: "", - isError: false, - }; - } catch (error: any) { - return { - // eslint-disable-next-line @typescript-eslint/naming-convention - "prompt-hash": "", - user: "", - date: "", - response: `Error: ${error.message}`, - // eslint-disable-next-line @typescript-eslint/naming-convention - finish_reason: "error", - isError: true, - }; - } - } - - async logInsert(contexts: string[] | undefined, request: string, response: string, parent: string | undefined): Promise { - try { - // build log data - const llmModelData = await ApiKeyManager.llmModel(); - const contextContentList = await this.loadContextsFromFiles(contexts); - const contextWithRoleList = contextContentList.map(content => { - return { - "role": "system", - "content": `${content}` - }; - }); - - let logData = { - "model": llmModelData?.model || "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": request - }, - { - "role": "assistant", - "content": response - }, - ...contextWithRoleList - ], - "timestamp": Math.floor(Date.now()/1000), - // eslint-disable-next-line @typescript-eslint/naming-convention - "request_tokens": 1, - // eslint-disable-next-line @typescript-eslint/naming-convention - "response_tokens": 1, - ...parent? {"parent": parent} : {} - }; - const insertValue = JSON.stringify(logData); - let insertValueOrFile = insertValue; - if (insertValue.length > 4 * 1024) { - const tempDir = os.tmpdir(); - const tempFile = path.join(tempDir, 'devchat_log_insert.json'); - await fs.promises.writeFile(tempFile, insertValue); - insertValueOrFile = tempFile; - } - - // build args for log insert - const args = ["-m", "devchat", "log", "--insert", insertValueOrFile]; - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - assertValue(stdout.indexOf('Failed to insert log') >= 0, stdout); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - const logs = JSON.parse(stdout.trim()).reverse(); - for (const log of logs) { - log.response = log.responses[0]; - delete log.responses; - } - return logs; - } catch (error: any) { - logger.channel()?.error(`Failed to insert log: ${error.message}`); - logger.channel()?.show(); - return []; - } - } - - async delete(hash: string): Promise { - try { - // build args for log delete - const args = ["-m", "devchat", "log", "--delete", hash]; - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - assertValue(stdout.indexOf('Failed to delete prompt') >= 0, stdout); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - return true; - } catch (error: any) { - logger.channel()?.error(`Failed to delete log: ${error.message}`); - logger.channel()?.show(); - return false; - } - } - - async log(options: LogOptions = {}): Promise { - try { - const args = this.buildLogArgs(options); - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - const logs = JSON.parse(stdout.trim()).reverse(); - for (const log of logs) { - log.response = log.responses[0]; - delete log.responses; - } - return logs; - } catch (error: any) { - logger.channel()?.error(`Failed to get logs: ${error.message}`); - logger.channel()?.show(); - return []; - } - } - - async loadRecommendCommands(): Promise { - try { - const args = ["-m", "devchat", "workflow", "config", "--json"]; - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - let workflowConfig; - try { - workflowConfig = JSON.parse(stdout.trim()); - } catch (error) { - logger.channel()?.error('Failed to parse commands JSON:', error); - return []; - } - - return workflowConfig.recommend?.workflows || []; - } catch (error: any) { - logger.channel()?.error(`Error: ${error.message}`); - logger.channel()?.show(); - return []; - } - } - - async commands(): Promise { - try { - const args = ["-m", "devchat", "workflow", "list", "--json"]; - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - let commands; - try { - commands = JSON.parse(stdout.trim()); - } catch (error) { - logger.channel()?.error('Failed to parse commands JSON:', error); - return []; - } - - // 确保每个CommandEntry对象的recommend字段默认为-1 - const recommendCommands = await this.loadRecommendCommands(); - commands = commands.map((cmd: any) => ({ - ...cmd, - recommend: recommendCommands.indexOf(cmd.name), - })); - - return commands; - } catch (error: any) { - logger.channel()?.error(`Error: ${error.message}`); - logger.channel()?.show(); - return []; - } - } - - async updateSysCommand(): Promise { - try { - const args = ["-m", "devchat", "workflow", "update"]; - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - logger.channel()?.trace(`${stdout}`); - return stdout; - } catch (error: any) { - logger.channel()?.error(`Error: ${error.message}`); - logger.channel()?.show(); - return ""; - } - } - - async topics(): Promise { - try { - const args = ["-m", "devchat", "topic", "-l"]; - - const {code, stdout, stderr} = await this.runCommand(args); - - assertValue(code !== 0, stderr || `Command exited with ${code}`); - if (stderr.trim() !== "") { - logger.channel()?.warn(`${stderr}`); - } - - const topics = JSON.parse(stdout.trim()).reverse(); - for (const topic of topics) { - if (topic.root_prompt.responses) { - topic.root_prompt.response = topic.root_prompt.responses[0]; - delete topic.root_prompt.responses; - } - } - return topics; - } catch (error: any) { - logger.channel()?.error(`Error: ${error.message}`); - logger.channel()?.show(); - return []; - } - } -} - -export default DevChat; diff --git a/src/toolwrapper/devchatCLI.ts b/src/toolwrapper/devchatCLI.ts new file mode 100644 index 0000000..797712b --- /dev/null +++ b/src/toolwrapper/devchatCLI.ts @@ -0,0 +1,294 @@ +import * as dotenv from "dotenv"; +import * as path from "path"; +import * as fs from "fs"; +import * as os from "os"; + +import { logger } from "../util/logger"; +import { CommandRun } from "../util/commonUtil"; +import { UiUtilWrapper } from "../util/uiUtil"; +import { ApiKeyManager } from "../util/apiKey"; +import { assertValue } from "../util/check"; +import { getFileContent } from "../util/commonUtil"; +import { DevChatConfig } from "../util/config"; + +import { getMicromambaUrl } from "../util/python_installer/conda_url"; + +const readFileAsync = fs.promises.readFile; + +const envPath = path.join(__dirname, "..", ".env"); +dotenv.config({ path: envPath }); + +export interface ChatOptions { + parent?: string; + reference?: string[]; + header?: string[]; + context?: string[]; +} + +export interface ChatResponse { + // eslint-disable-next-line @typescript-eslint/naming-convention + "prompt-hash": string; + user: string; + date: string; + response: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + finish_reason: string; + isError: boolean; +} + +export class DevChatCLI { + private commandRun: CommandRun; + + constructor() { + this.commandRun = new CommandRun(); + } + + private async buildArgs(options: ChatOptions): Promise { + let args = ["-m", "devchat", "route"]; + + if (options.reference) { + for (const reference of options.reference) { + args.push("-r", reference); + } + } + if (options.header) { + for (const header of options.header) { + args.push("-i", header); + } + } + if (options.context) { + for (const context of options.context) { + args.push("-c", context); + } + } + + if (options.parent) { + args.push("-p", options.parent); + } + + const llmModelData = await ApiKeyManager.llmModel(); + assertValue( + !llmModelData || !llmModelData.model, + "You must select a LLM model to use for conversations" + ); + args.push("-m", llmModelData.model); + + const functionCalling = DevChatConfig.getInstance().get( + "enable_function_calling" + ); + if (functionCalling) { + args.push("-a"); + } + + return args; + } + + private parseOutData(stdout: string, isPartial: boolean): ChatResponse { + const responseLines = stdout.trim().split("\n"); + + if (responseLines.length < 2) { + return this.createChatResponse("", "", "", "", !isPartial); + } + // logger.channel()?.info(`\n-responseLines: ${responseLines}`); + + const [userLine, remainingLines1] = this.extractLine( + responseLines, + "User: " + ); + const user = this.parseLine(userLine, /User: (.+)/); + + const [dateLine, remainingLines2] = this.extractLine( + remainingLines1, + "Date: " + ); + const date = this.parseLine(dateLine, /Date: (.+)/); + + const [promptHashLine, remainingLines3] = this.extractLine( + remainingLines2, + "prompt" + ); + const [finishReasonLine, remainingLines4] = this.extractLine( + remainingLines3, + "finish_reason:" + ); + + if (!promptHashLine) { + return this.createChatResponse( + "", + user, + date, + remainingLines4.join("\n"), + !isPartial + ); + } + + const finishReason = finishReasonLine.split(" ")[1]; + const promptHash = promptHashLine.split(" ")[1]; + const response = remainingLines4.join("\n"); + + return this.createChatResponse( + promptHash, + user, + date, + response, + false, + finishReason + ); + } + + private extractLine( + lines: string[], + startWith: string + ): [string, string[]] { + const index = lines.findIndex((line) => line.startsWith(startWith)); + const extractedLine = index !== -1 ? lines.splice(index, 1)[0] : ""; + return [extractedLine, lines]; + } + + private parseLine(line: string, regex: RegExp): string { + return line.match(regex)?.[1] ?? ""; + } + + private createChatResponse( + promptHash: string, + user: string, + date: string, + response: string, + isError: boolean, + finishReason = "" + ): ChatResponse { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + "prompt-hash": promptHash, + user, + date, + response, + // eslint-disable-next-line @typescript-eslint/naming-convention + finish_reason: finishReason, + isError, + }; + } + + public input(data: string) { + this.commandRun?.write(data + "\n"); + } + + public stop() { + this.commandRun.stop(); + } + + async runWorkflow( + content: string, + options: ChatOptions = {}, + onData: (data: ChatResponse) => void, + ): Promise { + // TODO: Use another cli command to run workflow instead of `devchat route` + try { + // build args for devchat prompt command + const args = await this.buildArgs(options); + args.push("--"); + args.push(content); + + // build env variables for prompt command + const llmModelData = await ApiKeyManager.llmModel(); + assertValue(!llmModelData, "No valid llm model selected"); + const envs = { + ...process.env, + // eslint-disable-next-line @typescript-eslint/naming-convention + PYTHONUTF8: 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + command_python: + DevChatConfig.getInstance().get("python_for_commands") || + "", + // eslint-disable-next-line @typescript-eslint/naming-convention + PYTHONPATH: + UiUtilWrapper.extensionPath() + "/tools/site-packages", + // eslint-disable-next-line @typescript-eslint/naming-convention + OPENAI_API_KEY: llmModelData.api_key.trim(), + DEVCHAT_UNIT_TESTS_USE_USER_MODEL: 1, + // eslint-disable-next-line @typescript-eslint/naming-convention + ...(llmModelData.api_base + ? { + OPENAI_API_BASE: llmModelData.api_base, + OPENAI_BASE_URL: llmModelData.api_base, + } + : {}), + DEVCHAT_PROXY: + DevChatConfig.getInstance().get("DEVCHAT_PROXY") || "", + MAMBA_BIN_PATH: getMicromambaUrl(), + }; + + // build process options + const spawnAsyncOptions = { + maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB + cwd: UiUtilWrapper.workspaceFoldersFirstPath(), + env: envs, + }; + + logger + .channel() + ?.info( + `api_key: ${llmModelData.api_key.replace( + /^(.{4})(.*)(.{4})$/, + (_, first, middle, last) => + first + middle.replace(/./g, "*") + last + )}` + ); + logger.channel()?.info(`api_base: ${llmModelData.api_base}`); + + // run command + // handle stdout as steam mode + let receviedStdout = ""; + const onStdoutPartial = (stdout: string) => { + receviedStdout += stdout; + const data = this.parseOutData(receviedStdout, true); + onData(data); + }; + // run command + const pythonApp = + DevChatConfig.getInstance().get("python_for_chat") || "python3"; + logger + .channel() + ?.info(`Running devchat:${pythonApp} ${args.join(" ")}`); + const { + exitCode: code, + stdout, + stderr, + } = await this.commandRun.spawnAsync( + pythonApp, + args, + spawnAsyncOptions, + onStdoutPartial, + undefined, + undefined, + undefined + ); + // handle result + assertValue(code !== 0, stderr || "Command exited with error code"); + const responseData = this.parseOutData(stdout, false); + // return result + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + "prompt-hash": "", + user: "", + date: "", + response: responseData.response, + // eslint-disable-next-line @typescript-eslint/naming-convention + finish_reason: "", + isError: false, + }; + } catch (error: any) { + return { + // eslint-disable-next-line @typescript-eslint/naming-convention + "prompt-hash": "", + user: "", + date: "", + response: `Error: ${error.message}`, + // eslint-disable-next-line @typescript-eslint/naming-convention + finish_reason: "error", + isError: true, + }; + } + } +} + diff --git a/src/toolwrapper/devchatClient.ts b/src/toolwrapper/devchatClient.ts new file mode 100644 index 0000000..a599ccd --- /dev/null +++ b/src/toolwrapper/devchatClient.ts @@ -0,0 +1,475 @@ +import axios, { AxiosResponse, CancelTokenSource } from "axios"; +import * as path from "path"; +import * as fs from "fs"; +import * as os from "os"; + +import { logger } from "../util/logger"; +import { getFileContent } from "../util/commonUtil"; + +import { UiUtilWrapper } from "../util/uiUtil"; + + +class DCLocalServicePortNotSetError extends Error { + constructor() { + super("DC_LOCALSERVICE_PORT is not set"); + this.name = "DCLocalServicePortNotSetError"; + } +} + +function timeThis( + target: Object, + propertyKey: string, + descriptor: TypedPropertyDescriptor +) { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: any[]) { + const start = process.hrtime.bigint(); + const result = await originalMethod.apply(this, args); + const end = process.hrtime.bigint(); + const nanoseconds = end - start; + const seconds = Number(nanoseconds) / 1e9; + + const className = target.constructor.name; + logger + .channel() + ?.debug(`Exec time [${className}.${propertyKey}]: ${seconds} s`); + return result; + }; + + return descriptor; +} + +function catchAndReturn(defaultReturn: any) { + return function ( + target: Object, + propertyKey: string, + descriptor: TypedPropertyDescriptor + ) { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: any[]) { + try { + return await originalMethod.apply(this, args); + } catch (error) { + if (error instanceof DCLocalServicePortNotSetError) { + logger.channel()?.warn(`DC_LOCALSERVICE_PORT is not set in [${propertyKey}]`); + return defaultReturn; + } + + logger.channel()?.error(`Error in [${propertyKey}]: ${error}`); + return defaultReturn; + } + }; + + return descriptor; + }; +} + +export interface ChatRequest { + content: string; + model_name: string; + api_key: string; + api_base: string; + parent?: string; + context?: string[]; +} + +export interface ChatResponse { + // eslint-disable-next-line @typescript-eslint/naming-convention + "prompt-hash": string; + user: string; + date: string; + response: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + finish_reason: string; + isError: boolean; + extra?: object; +} + +export interface LogData { + model: string; + messages: object[]; + parent?: string; + timestamp: number; + request_tokens: number; + response_tokens: number; +} + +export interface LogInsertRes { + hash?: string; +} + +export interface LogDeleteRes { + success?: boolean; +} + +export interface ShortLog { + hash: string; + parent: string | null; + user: string; + date: string; + request: string; + responses: string[]; + context: Array<{ + content: string; + role: string; + }>; +} + +export async function buildRoleContextsFromFiles( + files: string[] | undefined +): Promise { + const contexts: object[] = []; + if (!files) { + return contexts; + } + + for (const file of files) { + const content = await getFileContent(file); + + if (!content) { + continue; + } + contexts.push({ + role: "system", + content: `${content}`, + }); + } + return contexts; +} + +export class DevChatClient { + private baseURL: string | undefined; + + private _cancelMessageToken: CancelTokenSource | null = null; + + static readonly logRawDataSizeLimit = 4 * 1024; + + constructor() { + } + + async _get(path: string, config?: any): Promise { + if (!this.baseURL) { + if (!process.env.DC_LOCALSERVICE_PORT) { + logger.channel()?.info("No local service port found."); + throw new DCLocalServicePortNotSetError(); + } + const port: number = parseInt(process.env.DC_LOCALSERVICE_PORT || '8008', 10); + this.baseURL = `http://localhost:${port}`; + } + + try { + logger.channel()?.debug(`GET request to ${this.baseURL}${path}`); + const response = await axios.get(`${this.baseURL}${path}`, config); + return response; + } catch (error) { + console.error(error); + throw error; + } + } + async _post(path: string, data: any = undefined): Promise { + if (!this.baseURL) { + if (!process.env.DC_LOCALSERVICE_PORT) { + logger.channel()?.info("No local service port found."); + throw new DCLocalServicePortNotSetError(); + } + const port: number = parseInt(process.env.DC_LOCALSERVICE_PORT || '8008', 10); + this.baseURL = `http://localhost:${port}`; + } + + try { + logger.channel()?.debug(`POST request to ${this.baseURL}${path}`); + const response = await axios.post(`${this.baseURL}${path}`, data); + return response; + } catch (error) { + console.error(error); + throw error; + } + } + + @timeThis + @catchAndReturn([]) + async getWorkflowList(): Promise { + const response = await this._get("/workflows/list"); + logger + .channel() + ?.trace( + `getWorkflowList response data: \n${JSON.stringify( + response.data + )}` + ); + return response.data; + } + + @timeThis + @catchAndReturn({}) + async getWorkflowConfig(): Promise { + const response = await this._get("/workflows/config"); + logger + .channel() + ?.trace( + `getWorkflowConfig response data: \n${JSON.stringify( + response.data + )}` + ); + return response.data; + } + + @timeThis + @catchAndReturn(undefined) + async updateWorkflows(): Promise { + const response = await this._post("/workflows/update"); + logger + .channel() + ?.trace( + `updateWorkflows response data: \n${JSON.stringify( + response.data + )}` + ); + } + + @timeThis + async message( + message: ChatRequest, + onData: (data: ChatResponse) => void + ): Promise { + if (!this.baseURL) { + if (!process.env.DC_LOCALSERVICE_PORT) { + logger.channel()?.info("No local service port found."); + } + const port: number = parseInt(process.env.DC_LOCALSERVICE_PORT || '8008', 10); + this.baseURL = `http://localhost:${port}`; + } + + this._cancelMessageToken = axios.CancelToken.source(); + const workspace = UiUtilWrapper.workspaceFoldersFirstPath(); + // const workspace = undefined; + + const data = { + ...message, + workspace: workspace, + }; + + return new Promise(async (resolve, reject) => { + try { + const response = await axios.post( + `${this.baseURL}/message/msg`, + data, + { + responseType: "stream", + cancelToken: this._cancelMessageToken!.token, + } + ); + const chatRes: ChatResponse = { + "prompt-hash": "", // prompt-hash is not in chatting response, it is created in the later insertLog() + user: "", + date: "", + response: "", + finish_reason: "", + isError: false, + }; + + response.data.on("data", (chunk) => { + const chunkData = JSON.parse(chunk.toString()); + + if (chatRes.user === "") { + chatRes.user = chunkData["user"]; + } + if (chatRes.date === "") { + chatRes.date = chunkData["date"]; + } + chatRes.finish_reason = chunkData["finish_reason"]; + if (chatRes.finish_reason === "should_run_workflow") { + chatRes.extra = chunkData["extra"]; + logger.channel()?.debug("should run workflow via cli."); + return; + } + + chatRes.isError = chunkData["isError"]; + + chatRes.response += chunkData["content"]; + onData(chatRes); + }); + + response.data.on("end", () => { + resolve(chatRes); // Resolve the promise with chatRes when the stream ends + }); + + response.data.on("error", (error) => { + logger.channel()?.error("Streaming error:", error); + chatRes.isError = true; + chatRes.response += `\n${error}`; + resolve(chatRes); // handle error by resolving the promise + }); + } catch (error) { + const errorRes: ChatResponse = { + "prompt-hash": "", + user: "", + date: "", + response: `${error}`, + finish_reason: "", + isError: true, + }; + resolve(errorRes); // handle error by resolving the promise using an errorRes + } + }); + } + + cancelMessage(): void { + if (this._cancelMessageToken) { + this._cancelMessageToken.cancel( + "Message request cancelled by user" + ); + this._cancelMessageToken = null; + } + } + + /** + * Insert a message log. + * + * @param logData - The log data to be inserted. + * @returns A tuple of inserted hash and error message. + */ + @timeThis + @catchAndReturn({ hash: "" }) + async insertLog(logData: LogData): Promise { + let body = { + workspace: UiUtilWrapper.workspaceFoldersFirstPath(), + }; + + const jsondata = JSON.stringify(logData); + let filepath = ""; + + if (jsondata.length <= DevChatClient.logRawDataSizeLimit) { + // Use json data directly + body["jsondata"] = jsondata; + } else { + // Write json data to a temp file + const tempDir = os.tmpdir(); + const tempFile = path.join(tempDir, "devchat_log_insert.json"); + await fs.promises.writeFile(tempFile, jsondata); + filepath = tempFile; + body["filepath"] = filepath; + } + + const response = await this._post("/logs/insert", body); + logger + .channel() + ?.trace( + `insertLog response data: ${JSON.stringify( + response.data + )}, ${typeof response.data}}` + ); + + // Clean up temp file + if (filepath) { + try { + await fs.promises.unlink(filepath); + } catch (error) { + logger + .channel() + ?.error(`Failed to delete temp file ${filepath}: ${error}`); + } + } + + const res: LogInsertRes = { + hash: response.data["hash"], + }; + return res; + } + + @timeThis + @catchAndReturn({ success: false }) + async deleteLog(logHash: string): Promise { + const data = { + workspace: UiUtilWrapper.workspaceFoldersFirstPath(), + hash: logHash, + }; + const response = await this._post("/logs/delete", data); + logger + .channel() + ?.trace( + `deleteLog response data: ${JSON.stringify( + response.data + )}, ${typeof response.data}}` + ); + + const res: LogDeleteRes = { + success: response.data["success"], + }; + return res; + } + + @timeThis + @catchAndReturn([]) + async getTopicLogs( + topicRootHash: string, + limit: number, + offset: number + ): Promise { + const data = { + limit: limit, + offset: offset, + workspace: UiUtilWrapper.workspaceFoldersFirstPath(), + }; + const response = await this._get(`/topics/${topicRootHash}/logs`, { + params: data, + }); + + const logs: ShortLog[] = response.data; + logs.reverse(); + + logger + .channel() + ?.trace(`getTopicLogs response data: ${JSON.stringify(logs)}`); + + return logs; + } + + @timeThis + @catchAndReturn([]) + async getTopics(limit: number, offset: number): Promise { + const data = { + limit: limit, + offset: offset, + workspace: UiUtilWrapper.workspaceFoldersFirstPath(), + }; + const response = await this._get(`/topics`, { + params: data, + }); + + const topics: any[] = response.data; + topics.reverse(); + + logger + .channel() + ?.trace(`getTopics response data: ${JSON.stringify(topics)}`); + + return topics; + } + + @timeThis + @catchAndReturn(undefined) + async deleteTopic(topicRootHash: string): Promise { + const data = { + topic_hash: topicRootHash, + workspace: UiUtilWrapper.workspaceFoldersFirstPath(), + }; + + const response = await this._post("/topics/delete", data); + + logger + .channel() + ?.trace( + `deleteTopic response data: ${JSON.stringify(response.data)}` + ); + + return; + } + + stopAllRequest(): void { + this.cancelMessage(); + // add other requests here if needed + } +} \ No newline at end of file diff --git a/src/util/apiKey.ts b/src/util/apiKey.ts index eece373..e7a5046 100644 --- a/src/util/apiKey.ts +++ b/src/util/apiKey.ts @@ -1,7 +1,5 @@ // src/apiKey.ts -import DevChat from '@/toolwrapper/devchat'; -import { UiUtilWrapper } from './uiUtil'; import { DevChatConfig } from './config'; import { logger } from './logger'; diff --git a/src/util/findServicePort.ts b/src/util/findServicePort.ts new file mode 100644 index 0000000..a94be06 --- /dev/null +++ b/src/util/findServicePort.ts @@ -0,0 +1,26 @@ +import net from 'net'; + +export async function findAvailablePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer().listen(); + + server.on('listening', () => { + const address = server.address(); + if (typeof address !== 'object' || !address?.port) { + server.close(); + reject(new Error('Failed to get port from server')); + return; + } + server.close(() => resolve(address.port)); + }); + + server.on('error', (err) => { + const errWithCode = err as NodeJS.ErrnoException; + if (errWithCode.code === 'EADDRINUSE') { + reject(new Error('Port already in use')); + } else { + reject(err); + } + }); + }); +} diff --git a/src/util/localService.ts b/src/util/localService.ts new file mode 100644 index 0000000..6b5ee53 --- /dev/null +++ b/src/util/localService.ts @@ -0,0 +1,105 @@ +import { spawn, ChildProcess } from 'child_process'; +import { findAvailablePort } from './findServicePort'; +import * as http from 'http'; +import { logger } from './logger'; +import { DevChatConfig } from './config'; + +let serviceProcess: ChildProcess | null = null; + +export async function startLocalService(extensionPath: string, workspacePath: string): Promise { + if (serviceProcess) { + throw new Error('Local service is already running'); + } + + try { + // 1. 获取可用端口号 + const port = await findAvailablePort(); + + // 2. 设置环境变量 DC_SVC_PORT + process.env.DC_SVC_PORT = port.toString(); + + // 3. 设置 DC_SVC_WORKSPACE 环境变量 + process.env.DC_SVC_WORKSPACE = workspacePath; + + // 新增：设置 PYTHONPATH 环境变量 + process.env.PYTHONPATH = `${extensionPath}/tools/site-packages`; + + // 4. 启动进程 python main.py + const mainPyPath = extensionPath + "/tools/site-packages/devchat/_service/main.py"; + const pythonApp = + DevChatConfig.getInstance().get("python_for_chat") || "python3"; + serviceProcess = spawn(pythonApp, [mainPyPath], { + env: { ...process.env }, + stdio: 'inherit', + windowsHide: true, // hide the console window on Windows + }); + + serviceProcess.on('error', (err) => { + logger.channel()?.error('Failed to start local service:', err); + serviceProcess = null; + }); + + serviceProcess.on('exit', (code) => { + logger.channel()?.info(`Local service exited with code ${code}`); + serviceProcess = null; + }); + + // 5. 等待服务启动并验证 + await waitForServiceToStart(port); + + // 6. 服务启动成功后，记录启动的端口号到环境变量 + process.env.DC_LOCALSERVICE_PORT = port.toString(); + logger.channel()?.info(`Local service port recorded: ${port}`); + + return port; + } catch (error) { + logger.channel()?.error('Error starting local service:', error); + throw error; + } +} + +async function waitForServiceToStart(port: number): Promise { + const maxRetries = 30; + const retryInterval = 1000; // 1 second + + for (let i = 0; i < maxRetries; i++) { + try { + const response = await new Promise((resolve, reject) => { + http.get(`http://localhost:${port}/ping`, (res) => { + let data = ''; + res.on('data', (chunk) => data += chunk); + res.on('end', () => resolve(data)); + }).on('error', reject); + }); + + if (response === '{"message":"pong"}') { + logger.channel()?.info('Local service started successfully'); + return; + } + } catch (error) { + // Ignore errors and continue retrying + } + + await new Promise(resolve => setTimeout(resolve, retryInterval)); + } + + throw new Error('Failed to start local service: timeout'); +} + +export async function stopLocalService(): Promise { + return new Promise((resolve) => { + if (!serviceProcess) { + logger.channel()?.warn('No local service is running'); + resolve(); + return; + } + + serviceProcess.on('exit', () => { + serviceProcess = null; + logger.channel()?.info('Local service stopped'); + resolve(); + }); + + serviceProcess.kill(); + }); +} \ No newline at end of file diff --git a/test/handler/sendMessageBase.test.ts b/test/handler/sendMessageBase.test.ts deleted file mode 100644 index 4e638d3..0000000 --- a/test/handler/sendMessageBase.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { expect } from 'chai'; -// import { describe, it } from 'mocha'; -import sinon from 'sinon'; -import * as path from 'path'; -import { parseMessage, parseMessageAndSetOptions, processChatResponse, sendMessageBase, stopDevChatBase } from '../../src/handler/sendMessageBase'; -import { ChatResponse } from '../../src/toolwrapper/devchat'; -import { UiUtilWrapper } from '../../src/util/uiUtil'; - -import * as dotenv from 'dotenv'; - -const envPath = path.join(__dirname, '../../', '.env'); -dotenv.config({ path: envPath }); - -describe('sendMessageBase', () => { - let workspaceFoldersFirstPathStub: sinon.SinonStub; - let getConfigurationStub: sinon.SinonStub; - - beforeEach(() => { - workspaceFoldersFirstPathStub = sinon.stub(UiUtilWrapper, 'workspaceFoldersFirstPath'); - getConfigurationStub = sinon.stub(UiUtilWrapper, 'getConfiguration'); - }); - - afterEach(() => { - workspaceFoldersFirstPathStub.restore(); - getConfigurationStub.restore(); - }); - - describe('parseMessage', () => { - it('should parse message correctly', () => { - const message = '[context|path/to/context] [instruction|path/to/instruction] [reference|path/to/reference] Hello, world!'; - const result = parseMessage(message); - - expect(result.context).to.deep.equal(['path/to/context']); - expect(result.instruction).to.deep.equal(['path/to/instruction']); - expect(result.reference).to.deep.equal(['path/to/reference']); - expect(result.text).to.equal('Hello, world!'); - }); - }); - - describe('parseMessageAndSetOptions', () => { - it('should parse message and set options correctly', async () => { - const message = { - text: '[context|path/to/context] [instruction|path/to/instruction] [reference|path/to/reference] Hello, world!' - }; - - const [result, chatOptions] = await parseMessageAndSetOptions(message); - - expect(result.context).to.deep.equal(['path/to/context']); - expect(result.instruction).to.deep.equal(['path/to/instruction']); - expect(result.reference).to.deep.equal(['path/to/reference']); - expect(result.text).to.equal('Hello, world!'); - expect(chatOptions.context).to.deep.equal(['path/to/context']); - expect(chatOptions.header).to.deep.equal([]); - expect(chatOptions.reference).to.deep.equal(['path/to/reference']); - }); - }); - - describe('processChatResponse', () => { - it('should handle response text correctly when isError is false', async () => { - const partialDataText = 'Partial data'; - const chatResponse: ChatResponse = { - "finish_reason": "", - response: 'Hello, user!', - isError: false, - user: 'user', - date: '2022-01-01T00:00:00.000Z', - 'prompt-hash': 'responsehash' - }; - - const result = await processChatResponse(chatResponse); - expect(result).to.equal('Hello, user!'); - }); - - it('should handle response text correctly when isError is true', async () => { - const partialDataText = 'Partial data'; - const chatResponse: ChatResponse = { - "finish_reason": "", - response: 'Error occurred!', - isError: true, - user: 'user', - date: '2022-01-01T00:00:00.000Z', - 'prompt-hash': 'responsehash' - }; - - const result = await processChatResponse(chatResponse); - expect(result).to.equal('Error occurred!'); - }); - }); - - describe('sendMessageBase', () => { - it('should send message correct with DevChat access key', async function() { - const message = { - text: 'Hello, world!' - }; - const handlePartialData = (data: { command: string, text: string, user: string, date: string }) => { - // Handle partial data - }; - - workspaceFoldersFirstPathStub.returns('./'); - - getConfigurationStub.withArgs('DevChat', 'Access_Key_DevChat').returns(process.env.TEST_DEVCHAT_KEY); - getConfigurationStub.withArgs('DevChat', 'OpenAI.temperature').returns(0); - getConfigurationStub.withArgs('DevChat', 'OpenAI.stream').returns('true'); - - const result = await sendMessageBase(message, handlePartialData); - expect(result).to.be.an('object'); - expect(result!.command).to.equal('receiveMessage'); - expect(result!.text).to.be.a('string'); - expect(result!.hash).to.be.a('string'); - expect(result!.user).to.be.a('string'); - expect(result!.date).to.be.a('string'); - // TODO fix - // Need to mock more config setting - // expect(result!.isError).to.be.false; - }); - - it('should send message error with invalid api key', async function() { - const message = { - text: 'Hello, world!' - }; - const handlePartialData = (data: { command: string, text: string, user: string, date: string }) => { - // Handle partial data - }; - - workspaceFoldersFirstPathStub.returns('./'); - - getConfigurationStub.withArgs('DevChat', 'Access_Key_DevChat').returns('sk-KvH7ZCtHmFDCBTqH0jUv'); - getConfigurationStub.withArgs('DevChat', 'OpenAI.temperature').returns('0'); - getConfigurationStub.withArgs('DevChat', 'OpenAI.stream').returns('true'); - - const result = await sendMessageBase(message, handlePartialData); - expect(result).to.be.an('object'); - expect(result!.command).to.equal('receiveMessage'); - expect(result!.text).to.be.a('string'); - expect(result!.hash).to.be.a('string'); - expect(result!.user).to.be.a('string'); - expect(result!.date).to.be.a('string'); - expect(result!.isError).to.be.true; - }); - }); - - describe('stopDevChatBase', () => { - it('should stop sendMessageBase correctly', async () => { - const message = { - text: 'Hello, world!' - }; - const handlePartialData = (data: { command: string, text: string, user: string, date: string }) => { - // Handle partial data - }; - - workspaceFoldersFirstPathStub.returns('./'); - - getConfigurationStub.withArgs('DevChat', 'Access_Key_DevChat').returns(process.env.TEST_DEVCHAT_KEY); - getConfigurationStub.withArgs('DevChat', 'OpenAI.temperature').returns(0); - getConfigurationStub.withArgs('DevChat', 'OpenAI.stream').returns('true'); - - - // Start sendMessageBase in a separate Promise - const sendMessagePromise = sendMessageBase(message, handlePartialData); - - // Wait for a short period to ensure sendMessageBase has started - await new Promise((resolve) => setTimeout(resolve, 100)); - - // Call stopDevChatBase - const stopMessage = { - text: 'stop' - }; - await stopDevChatBase(stopMessage); - - // Check if sendMessageBase has been stopped and returns an error - // TODO fix - // Need to mock more config setting - // try { - // const result = await sendMessagePromise; - // expect(result).to.undefined; - // } catch (error) { - // expect(error).to.be.an('error'); - // } - }); - }); -}); \ No newline at end of file diff --git a/test/toolwrapper/devchat.test.ts b/test/toolwrapper/devchat.test.ts deleted file mode 100644 index a77e224..0000000 --- a/test/toolwrapper/devchat.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { expect } from 'chai'; -// import { describe, it } from 'mocha'; -import sinon from 'sinon'; -import DevChat, { ChatOptions } from '../../src/toolwrapper/devchat'; -import { CommandRun } from '../../src/util/commonUtil'; -import { UiUtilWrapper } from '../../src/util/uiUtil'; -import { ApiKeyManager } from '../../src/util/apiKey'; - - -describe('DevChat', () => { - let devChat: DevChat; - let spawnAsyncStub: sinon.SinonStub; - let workspaceFoldersFirstPathStub: sinon.SinonStub; - let apiKeyManagerStub: sinon.SinonStub; - - beforeEach(() => { - devChat = new DevChat(); - spawnAsyncStub = sinon.stub(CommandRun.prototype, 'spawnAsync'); - workspaceFoldersFirstPathStub = sinon.stub(UiUtilWrapper, 'workspaceFoldersFirstPath'); - apiKeyManagerStub = sinon.stub(ApiKeyManager, 'llmModel'); - }); - - afterEach(() => { - spawnAsyncStub.restore(); - workspaceFoldersFirstPathStub.restore(); - apiKeyManagerStub.restore(); - }); - - - describe('chat', () => { - it('should return a ChatResponse object with isError false when the chat is successful', async () => { - const content = 'Test chat content'; - const options: ChatOptions = { - // Provide mock values for the options - parent: 'parent_value', - reference: ['ref1', 'ref2'], - header: ['header1', 'header2'], - context: ['context1', 'context2'], - }; - const mockResponse = { - exitCode: 0, - stdout: 'User: Test user\nDate: 2022-01-01\nTest chat response\nprompt-hash: 12345', - stderr: '', - }; - const mockWorkspacePath = './'; - const llmModelResponse = { - "model": "gpt-3.5-turbo", - "api_key": "DC.1234567890" - } - - spawnAsyncStub.resolves(mockResponse); - workspaceFoldersFirstPathStub.returns(mockWorkspacePath); - apiKeyManagerStub.resolves(llmModelResponse); - - const response = await devChat.chat(content, options, (data)=>{}, false); - - expect(response).to.have.property('prompt-hash', ''); - expect(response).to.have.property('user', ''); - expect(response).to.have.property('date', ''); - expect(response).to.have.property('response', 'Test chat response'); - expect(response).to.have.property('isError', false); - expect(spawnAsyncStub.calledOnce).to.be.true; - expect(workspaceFoldersFirstPathStub.calledOnce).to.be.true; - }); - - // Add more test cases for the chat method here - }); - - // ... other test cases -}); \ No newline at end of file diff --git a/test/util/findServicePort.test.ts b/test/util/findServicePort.test.ts new file mode 100644 index 0000000..8ff59e5 --- /dev/null +++ b/test/util/findServicePort.test.ts @@ -0,0 +1,29 @@ +// test/util/findServicePort.test.ts + +import { expect } from 'chai'; +import net from 'net'; +import { findAvailablePort } from '../../src/util/findServicePort'; + +describe('findAvailablePort', () => { + it('should return an available port when successful', async () => { + // Arrange + const expectedPort = await findAvailablePort(); + + // Act + const server = net.createServer(); + const isAvailable = await new Promise((resolve) => { + server.listen(expectedPort, () => { + server.close(); + resolve(true); + }); + server.on('error', () => { + resolve(false); + }); + }); + + // Assert + expect(isAvailable).to.be.true; + expect(expectedPort).to.be.a('number'); + expect(expectedPort).to.be.greaterThan(0); + }); +}); \ No newline at end of file diff --git a/test/util/localService.test.ts b/test/util/localService.test.ts new file mode 100644 index 0000000..1cd8cbc --- /dev/null +++ b/test/util/localService.test.ts @@ -0,0 +1,72 @@ +// test/util/localService.test.ts + +import { expect } from 'chai'; +import { startLocalService, stopLocalService } from '../../src/util/localService'; +import * as http from 'http'; + +describe('localService', () => { + let port: number; + + describe('startLocalService', () => { + it('should start the local service successfully', async () => { + const extensionPath = '.'; + const workspacePath = '.'; + + port = await startLocalService(extensionPath, workspacePath); + + expect(port).to.be.a('number'); + expect(process.env.DC_SVC_PORT).to.equal(port.toString()); + expect(process.env.DC_SVC_WORKSPACE).to.equal(workspacePath); + expect(process.env.DC_LOCALSERVICE_PORT).to.equal(port.toString()); + + // Verify that the service is running by sending a ping request + const response = await sendPingRequest(port); + expect(response).to.equal('{"message":"pong"}'); + }); + }); + + describe('stopLocalService', () => { + it('should stop the local service', async () => { + await stopLocalService(); + + // Wait a bit to ensure the service has fully stopped + await new Promise(resolve => setTimeout(resolve, 1000)); + + // Verify that the service is no longer running + try { + await sendPingRequest(port); + throw new Error('Service is still running'); + } catch (error) { + console.log('Error type:', typeof error); + console.log('Error:', error); + + if (error instanceof Error) { + expect(error.message).to.include('connect ECONNREFUSED'); + } else if (typeof error === 'object' && error !== null) { + // Check if the error object has a 'code' property + if ('code' in error) { + expect(error.code).to.equal('ECONNREFUSED'); + } else if ('errors' in error && Array.isArray(error.errors)) { + // Check if it's an AggregateError-like object + const hasConnectionRefused = error.errors.some((e: any) => e.code === 'ECONNREFUSED'); + expect(hasConnectionRefused).to.be.true; + } else { + throw new Error(`Unexpected error structure: ${JSON.stringify(error)}`); + } + } else { + throw new Error(`Unexpected error type: ${typeof error}`); + } + } + }); +}); +}); + +function sendPingRequest(port: number): Promise { + return new Promise((resolve, reject) => { + http.get(`http://localhost:${port}/ping`, (res) => { + let data = ''; + res.on('data', (chunk) => data += chunk); + res.on('end', () => resolve(data)); + }).on('error', reject); + }); +} \ No newline at end of file diff --git a/tools b/tools index 0557f55..c53f73d 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit 0557f55a8d5dadabcf225a2b7a24db9db120f895 +Subproject commit c53f73df6322df029075593792bd5ec111bfd9df diff --git a/tsconfig.json b/tsconfig.json index 603665b..f6ea4ef 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,7 +18,8 @@ "@/*": [ "./src/*" ] - } + }, + "experimentalDecorators": true }, "exclude": [ "test","gui","tools"