support workflow commands

This commit is contained in:
bobo.yang 2023-11-22 17:45:38 +08:00
parent 7dc8551ff4
commit bf8386c068
4 changed files with 334 additions and 127 deletions

1
.gitignore vendored
View File

@ -13,3 +13,4 @@ node_modules
.vscode/settings.json
workflows/

View File

@ -11,15 +11,18 @@ import { ApiKeyManager } from '../util/apiKey';
import { logger } from '../util/logger';
import { exec as execCb } from 'child_process';
import { promisify } from 'util';
import { CommandRun, createTempSubdirectory } from '../util/commonUtil';
import { CommandResult, CommandRun, createTempSubdirectory } from '../util/commonUtil';
import { WorkflowRunner } from './workflowExecutor';
import DevChat from '../toolwrapper/devchat';
const exec = promisify(execCb);
let askcode_stop = true;
let askcode_runner : CommandRun | null = null;
let askcodeRunner : CommandRun | null = null;
let commandRunner : WorkflowRunner | null = null;
let _lastMessage: any = undefined;
export function createTempFile(content: string): string {
// Generate a unique file name
const fileName = path.join(os.tmpdir(), `temp_${Date.now()}.txt`);
@ -35,121 +38,19 @@ export function deleteTempFiles(fileName: string): void {
fs.unlinkSync(fileName);
}
regInMessage({command: 'askCode', text: '', parent_hash: undefined});
regOutMessage({ command: 'receiveMessage', text: 'xxxx', hash: 'xxx', user: 'xxx', date: 'xxx'});
export async function askCode(message: any, panel: vscode.WebviewPanel|vscode.WebviewView): Promise<void> {
try {
askcode_stop = false;
askcode_runner = null;
_lastMessage = [message];
_lastMessage[0]['askCode'] = true;
const port = await UiUtilWrapper.getLSPBrigePort();
const pythonVirtualEnv: string | undefined = vscode.workspace.getConfiguration('DevChat').get('PythonVirtualEnv');
if (!pythonVirtualEnv) {
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: "Index code fail.", hash: "", user: "", date: 0, isError: true });
return ;
}
let envs = {
PYTHONUTF8:1,
...process.env,
};
const llmModelData = await ApiKeyManager.llmModel();
if (!llmModelData) {
logger.channel()?.error('No valid llm model is selected!');
logger.channel()?.show();
return;
}
let openaiApiKey = llmModelData.api_key;
if (!openaiApiKey) {
logger.channel()?.error('The OpenAI key is invalid!');
logger.channel()?.show();
return;
}
envs['OPENAI_API_KEY'] = openaiApiKey;
const openAiApiBase = llmModelData.api_base;
if (openAiApiBase) {
envs['OPENAI_API_BASE'] = openAiApiBase;
}
const workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath();
if (askcode_stop) {
return;
}
try {
let outputResult = "";
askcode_runner = new CommandRun();
const command = pythonVirtualEnv.trim();
const args = [UiUtilWrapper.extensionPath() + "/tools/askcode_index_query.py", "query", message.text, `${port}`];
const result = await askcode_runner.spawnAsync(command, args, { env: envs, cwd: workspaceDir }, (data) => {
outputResult += data;
MessageHandler.sendMessage(panel, { command: 'receiveMessagePartial', text: outputResult, hash:"", user:"", isError: false });
logger.channel()?.info(data);
}, (data) => {
logger.channel()?.error(data);
}, undefined, undefined);
if (result.exitCode === 0) {
// save askcode result to devchat
const stepIndex = result.stdout.lastIndexOf("```Step");
const stepEndIndex = result.stdout.lastIndexOf("```");
let resultOut = result.stdout;
if (stepIndex > 0 && stepEndIndex > 0) {
resultOut = result.stdout.substring(stepEndIndex+3, result.stdout.length);
}
let logHash = await insertDevChatLog(message, "/ask-code " + message.text, resultOut);
if (!logHash) {
logHash = "";
logger.channel()?.error(`Failed to insert devchat log.`);
logger.channel()?.show();
}
MessageHandler.sendMessage(panel, { command: 'receiveMessagePartial', text: result.stdout, hash:logHash, user:"", isError: false });
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: result.stdout, hash:logHash, user:"", date:0, isError: false });
const dateStr = Math.floor(Date.now()/1000).toString();
await handleTopic(
message.parent_hash,
{"text": "/ask-code " + message.text},
{ response: result.stdout, "prompt-hash": logHash, user: "", "date": dateStr, finish_reason: "", isError: false });
} else {
logger.channel()?.info(`${result.stdout}`);
if (askcode_stop == false) {
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: result.stderr, hash: "", user: "", date: 0, isError: true });
}
}
} catch (error) {
if (error instanceof Error) {
logger.channel()?.error(`error: ${error.message}`);
} else {
logger.channel()?.error(`An unknown error occurred: ${error}`);
}
logger.channel()?.show();
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: "Did not get relevant context from AskCode.", hash: "", user: "", date: 0, isError: true });
}
} finally {
askcode_stop = true;
askcode_runner = null;
}
regInMessage({command: 'userInput', text: '{"field": "value", "field2": "value2"}'});;
export async function userInput(message: any, panel: vscode.WebviewPanel|vscode.WebviewView): Promise<void> {
commandRunner?.input(message.text);
}
// eslint-disable-next-line @typescript-eslint/naming-convention
regInMessage({command: 'sendMessage', text: '', parent_hash: undefined});
regOutMessage({ command: 'receiveMessage', text: 'xxxx', hash: 'xxx', user: 'xxx', date: 'xxx'});
regOutMessage({ command: 'receiveMessagePartial', text: 'xxxx', user: 'xxx', date: 'xxx'});
// message: { command: 'sendMessage', text: 'xxx', hash: 'xxx'}
// return message:
// { command: 'receiveMessage', text: 'xxxx', hash: 'xxx', user: 'xxx', date: 'xxx'}
// { command: 'receiveMessagePartial', text: 'xxxx', user: 'xxx', date: 'xxx'}
export async function sendMessage(message: any, panel: vscode.WebviewPanel|vscode.WebviewView, function_name: string|undefined = undefined): Promise<void> {
if (function_name !== undefined && function_name !== "") {
export async function sendMessage(message: any, panel: vscode.WebviewPanel|vscode.WebviewView, functionName: string|undefined = undefined): Promise<void> {
// check whether the message is a command
if (functionName !== undefined && functionName !== "") {
const messageText = _lastMessage[0].text.trim();
if (messageText[0] === '/' && message.text[0] !== '/') {
const indexS = messageText.indexOf(' ');
@ -161,7 +62,39 @@ export async function sendMessage(message: any, panel: vscode.WebviewPanel|vscod
message.text = preCommand + ' ' + message.text;
}
}
_lastMessage = [message, function_name];
_lastMessage = [message, functionName];
const messageText = message.text.trim();
if (messageText[0] === '/') {
// split messageText by ' ' or '\n' or '\t'
const messageTextArr = messageText.split(/ |\n|\t/);
// get command name from messageTextArr
const commandName = messageTextArr[0].substring(1);
// test whether the command is a execute command
const devChat = new DevChat();
const stdout = await devChat.commandPrompt(commandName);
// try parse stdout by json
let stdoutJson: any = null;
try {
stdoutJson = JSON.parse(stdout);
} catch (error) {
// do nothing
}
if (stdoutJson) {
// run command
try {
commandRunner = null;
commandRunner = new WorkflowRunner();
await commandRunner.run(commandName, stdoutJson, message, panel);
} finally {
commandRunner = null;
}
return ;
}
}
// Add a new field to store the names of temporary files
let tempFiles: string[] = [];
@ -192,7 +125,7 @@ export async function sendMessage(message: any, panel: vscode.WebviewPanel|vscod
const responseMessage = await sendMessageBase(message, (data: { command: string, text: string, user: string, date: string}) => {
MessageHandler.sendMessage(panel, data, false);
}, function_name);
}, functionName);
if (responseMessage) {
MessageHandler.sendMessage(panel, responseMessage);
}
@ -209,11 +142,7 @@ regInMessage({command: 'regeneration'});
export async function regeneration(message: any, panel: vscode.WebviewPanel|vscode.WebviewView): Promise<void> {
// call sendMessage to send last message again
if (_lastMessage) {
if (_lastMessage[0]['askCode']) {
await askCode(_lastMessage[0], panel);
} else {
await sendMessage(_lastMessage[0], panel, _lastMessage[1]);
}
await sendMessage(_lastMessage[0], panel, _lastMessage[1]);
}
}
@ -221,13 +150,9 @@ regInMessage({command: 'stopDevChat'});
export async function stopDevChat(message: any, panel: vscode.WebviewPanel|vscode.WebviewView): Promise<void> {
stopDevChatBase(message);
if (askcode_stop === false) {
askcode_stop = true;
if (askcode_runner) {
askcode_runner.stop();
askcode_runner = null;
}
await vscode.commands.executeCommand('DevChat.AskCodeIndexStop');
if (commandRunner) {
commandRunner.stop();
commandRunner = null;
}
}

View File

@ -0,0 +1,281 @@
// TODO
// 临时解决方案,后续需要修改
import * as vscode from 'vscode';
import { UiUtilWrapper } from "../util/uiUtil";
import { MessageHandler } from "../handler/messageHandler";
import { ApiKeyManager } from "../util/apiKey";
import { logger } from "../util/logger";
import { CommandResult, CommandRun, saveModelSettings } from "../util/commonUtil";
import { handleTopic, insertDevChatLog } from "./sendMessageBase";
import { regInMessage } from "@/util/reg_messages";
import parseArgsStringToArgv from 'string-argv';
async function handleWorkflowRequest(request): Promise<string | undefined> {
/*
request: {
"command": "some command",
"args": {
"arg1": "value1",
"arg2": "value2"
}
}
response: {
"status": "success",
"result": "success",
"detail": "some detail"
}
*/
if (!request || !request.command) {
return undefined;
}
if (request.command === "get_lsp_brige_port") {
return JSON.stringify({
"status": "success",
"result": await UiUtilWrapper.getLSPBrigePort()
});
} else {
return JSON.stringify({
"status": "fail",
"result": "fail",
"detail": "command is not supported"
});
}
}
// TODO
// 临时解决方案,后续需要修改
// 执行workflow
// workflow执行时都是通过启动一个进程的方式来执行。
// 与一般进程不同的是:
// 1. 通过UI交互可以停止该进程
// 2. 需要在进程启动前初始化相关的环境变量
// 3. 需要处理进程的通信
export class WorkflowRunner {
private _commandRunner: CommandRun | null = null;
private _stop: boolean = false;
private _cacheOut: string = "";
private _panel: vscode.WebviewPanel|vscode.WebviewView | null = null;
constructor() {}
private async _getApiKeyAndApiBase(): Promise<[string | undefined, string | undefined]> {
const llmModelData = await ApiKeyManager.llmModel();
if (!llmModelData) {
logger.channel()?.error('No valid llm model is selected!');
logger.channel()?.show();
return [undefined, undefined];
}
let openaiApiKey = llmModelData.api_key;
if (!openaiApiKey) {
logger.channel()?.error('The OpenAI key is invalid!');
logger.channel()?.show();
return [undefined, undefined];
}
const openAiApiBase = llmModelData.api_base;
return [openaiApiKey, openAiApiBase];
}
private _parseCommandOutput(outputStr: string): string {
/*
output is format as:
<<Start>>
{"content": "data"}
<<End>>
*/
const outputWitchCache = this._cacheOut + outputStr;
this._cacheOut = "";
let outputResult = "";
let curPos = 0;
while (true) {
const startPos = outputWitchCache.indexOf('<<Start>>', curPos);
const startPos2 = outputWitchCache.indexOf('```', curPos);
if (startPos === -1 && startPos2 === -1) {
break;
}
const isStart = (startPos2 === -1) || (startPos > -1 && startPos < startPos2);
let endPos = -1;
if (isStart) {
endPos = outputWitchCache.indexOf('<<End>>', startPos+9);
} else {
endPos = outputWitchCache.indexOf('```', startPos2+3);
}
if (endPos === -1) {
this._cacheOut = outputWitchCache.substring(startPos, outputWitchCache.length);
break;
}
let contentStr = "";
if (isStart) {
contentStr = outputWitchCache.substring(startPos+9, endPos);
curPos = endPos+7;
} else {
contentStr = outputWitchCache.substring(startPos2, endPos+3);
curPos = endPos+3;
}
outputResult += contentStr.trim() + "\n\n";
}
return outputResult;
}
private async _runCommand(commandWithArgs: string, commandEnvs: any): Promise<[CommandResult | undefined, string]> {
const workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath() || "";
let commandOutput = "";
let commandAnswer = "";
try {
const commandAndArgsList = parseArgsStringToArgv(commandWithArgs);
this._commandRunner = new CommandRun();
await saveModelSettings();
const result = await this._commandRunner.spawnAsync(commandAndArgsList[0], commandAndArgsList.slice(1), { env: commandEnvs, cwd: workspaceDir }, async (data) => {
// handle command stdout
const newData = this._parseCommandOutput(data);
// if newData is json string, then process it by handleWorkflowRequest
let newDataObj: any = undefined;
try {
newDataObj = JSON.parse(newData);
const result = await handleWorkflowRequest(newDataObj);
if (result) {
this.input(result);
} else if (newDataObj!.result) {
commandAnswer = newDataObj!.result;
commandOutput += newDataObj!.result;
logger.channel()?.info(newDataObj!.result);
MessageHandler.sendMessage(this._panel!, { command: 'receiveMessagePartial', text: commandOutput, hash:"", user:"", isError: false });
}
} catch (e) {
if (newData.length > 0){
commandOutput += newData;
logger.channel()?.info(newData);
MessageHandler.sendMessage(this._panel!, { command: 'receiveMessagePartial', text: commandOutput, hash:"", user:"", isError: false });
}
}
}, (data) => {
// handle command stderr
logger.channel()?.error(data);
logger.channel()?.show();
}, undefined, undefined);
return [result, commandAnswer];
} catch (error) {
if (error instanceof Error) {
logger.channel()?.error(`error: ${error.message}`);
} else {
logger.channel()?.error(`An unknown error occurred: ${error}`);
}
logger.channel()?.show();
}
return [undefined, ""];
}
public stop(): void {
this._stop = true;
if (this._commandRunner) {
this._commandRunner.stop();
this._commandRunner = null;
}
}
public input(data): void {
const userInputWithFlag = `\n<<Start>>\n${data}\n<<End>>\n`;
this._commandRunner?.write(userInputWithFlag);
}
public async run(workflow: string, commandDefines: any, message: any, panel: vscode.WebviewPanel|vscode.WebviewView): Promise<void> {
/*
1. workflow是否有输入存在
2. workflow的环境变量信息
3. workflow command
4. workflow command输出
*/
this._panel = panel;
// 获取workflow的python命令
const pythonVirtualEnv: string | undefined = vscode.workspace.getConfiguration('DevChat').get('PythonVirtualEnv');
if (!pythonVirtualEnv) {
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: "Index code fail.", hash: "", user: "", date: 0, isError: true });
return ;
}
// 获取扩展路径
const extensionPath = UiUtilWrapper.extensionPath();
// 获取api_key 和 api_base
const [apiKey, aipBase] = await this._getApiKeyAndApiBase();
if (!apiKey) {
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: "The OpenAI key is invalid!", hash: "", user: "", date: 0, isError: true });
return ;
}
// 构建子进程环境变量
const workflowEnvs = {
// eslint-disable-next-line @typescript-eslint/naming-convention
"PYTHONUTF8":1,
"DEVCHATPYTHON": UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3",
"PYTHONLIBPATH": `${extensionPath}/tools/site-packages`,
"PARENT_HASH": message.parent_hash,
...process.env,
// eslint-disable-next-line @typescript-eslint/naming-convention
OPENAI_API_KEY: apiKey,
// eslint-disable-next-line @typescript-eslint/naming-convention
...(aipBase ? { 'OPENAI_API_BASE': aipBase } : {})
};
const requireInput = Object.keys(commandDefines.parameters).filter(key => commandDefines.parameters[key].required === true).length > 0;
if (requireInput && message.text.replace("/" + workflow, "").trim() === "") {
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: `The workflow ${workflow} need input!`, hash: "", user: "", date: 0, isError: true });
return ;
}
const workflowCommand = commandDefines.steps[0].command.replace(
'{command_python}', `${pythonVirtualEnv}`).replace(
'{input}', `${message.text.replace("/" + workflow, "").trim()}`);
const [commandResult, commandAnswer] = await this._runCommand(workflowCommand, workflowEnvs);
if (commandResult && commandResult.exitCode === 0) {
const resultOut = commandAnswer === "" ? "success" : commandAnswer;
let logHash = await insertDevChatLog(message, message.text, resultOut);
if (!logHash) {
logHash = "";
logger.channel()?.error(`Failed to insert devchat log.`);
logger.channel()?.show();
}
//MessageHandler.sendMessage(panel, { command: 'receiveMessagePartial', text: resultOut, hash:logHash, user:"", isError: false });
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: resultOut, hash:logHash, user:"", date:0, isError: false });
const dateStr = Math.floor(Date.now()/1000).toString();
await handleTopic(
message.parent_hash,
{"text": "/ask-code " + message.text},
// eslint-disable-next-line @typescript-eslint/naming-convention
{ response: resultOut, "prompt-hash": logHash, user: "", "date": dateStr, finish_reason: "", isError: false });
} else if (commandResult) {
logger.channel()?.info(`${commandResult.stdout}`);
if (this._stop === false) {
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: commandResult.stderr, hash: "", user: "", date: 0, isError: true });
}
}
}
}

2
tools

@ -1 +1 @@
Subproject commit 9049c4aee8ed8329d1ad799134d98200cf149f8e
Subproject commit 5d49b6e4be35709b02dd4229ec9883b1bdace7ba