call smartqa
This commit is contained in:
bobo.yang
parent
cb2ea1deb6
commit
0886826da7
@ -8,6 +8,9 @@
|
||||
"engines": {
|
||||
"vscode": "^1.75.0"
|
||||
},
|
||||
"extensionDependencies": [
|
||||
"merico.lang-bridge-vsc"
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/devchat-ai/devchat-vscode.git"
|
||||
|
||||
@ -284,9 +284,6 @@ export function registerAskCodeIndexStartCommand(context: vscode.ExtensionContex
|
||||
if (!pythonVirtualEnv) {
|
||||
progressBar.update("Installing devchat-ask. See OUTPUT for progress...", 0);
|
||||
await installAskCode(supportedFileTypes, progressBar, indexCode);
|
||||
} else {
|
||||
progressBar.update("Index source files. See OUTPUT for progress...", 0);
|
||||
await indexCode(pythonVirtualEnv, supportedFileTypes, progressBar);
|
||||
}
|
||||
|
||||
updateIndexingStatus("stopped");
|
||||
@ -310,7 +307,7 @@ async function installAskCode(supportedFileTypes, progressBar: any, callback: Fu
|
||||
return;
|
||||
}
|
||||
|
||||
UiUtilWrapper.updateConfiguration("DevChat", "PythonVirtualEnv", pythonEnvPath.trim());
|
||||
await UiUtilWrapper.updateConfiguration("DevChat", "PythonVirtualEnv", pythonEnvPath.trim());
|
||||
logger.channel()?.info(`Installation finished.`);
|
||||
|
||||
// Execute the callback function after the installation is finished
|
||||
@ -318,68 +315,6 @@ async function installAskCode(supportedFileTypes, progressBar: any, callback: Fu
|
||||
}
|
||||
|
||||
async function indexCode(pythonVirtualEnv, supportedFileTypes, progressBar: any) {
|
||||
let envs = {};
|
||||
|
||||
const llmModelData = await ApiKeyManager.llmModel();
|
||||
if (!llmModelData) {
|
||||
logger.channel()?.error('No valid llm model is selected!');
|
||||
logger.channel()?.show();
|
||||
|
||||
progressBar.endWithError("No valid llm model is selected!");
|
||||
return;
|
||||
}
|
||||
|
||||
let openaiApiKey = llmModelData.api_key;
|
||||
if (!openaiApiKey) {
|
||||
logger.channel()?.error('The OpenAI key is invalid!');
|
||||
logger.channel()?.show();
|
||||
|
||||
progressBar.endWithError("The OpenAI key is invalid!");
|
||||
return;
|
||||
}
|
||||
envs['OPENAI_API_KEY'] = openaiApiKey;
|
||||
|
||||
const openAiApiBase = llmModelData.api_base;
|
||||
if (openAiApiBase) {
|
||||
envs['OPENAI_API_BASE'] = openAiApiBase;
|
||||
}
|
||||
|
||||
const workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath();
|
||||
|
||||
const command = pythonVirtualEnv.trim();
|
||||
const args = [UiUtilWrapper.extensionPath() + "/tools/askcode_index_query.py", "index", ".", supportedFileTypes];
|
||||
const options = { env: envs, cwd: workspaceDir };
|
||||
|
||||
indexProcess = new CommandRun();
|
||||
const result = await indexProcess.spawnAsync(command, args, options, (data) => {
|
||||
if (data.includes('Skip file:')) {
|
||||
return;
|
||||
}
|
||||
logger.channel()?.info(`${data}`);
|
||||
}, (data) => {
|
||||
if (data.includes('Skip file:')) {
|
||||
return;
|
||||
}
|
||||
logger.channel()?.info(`${data}`);
|
||||
}, undefined, undefined);
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
if (result.exitCode === null) {
|
||||
logger.channel()?.info(`Indexing stopped!`);
|
||||
progressBar.endWithError(`Indexing stopped!`);
|
||||
} else {
|
||||
logger.channel()?.error(`Indexing failed: ${result.stderr}`);
|
||||
logger.channel()?.show();
|
||||
progressBar.endWithError(`Indexing failed: ${result.stderr}`);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
updateLastModifyTime();
|
||||
logger.channel()?.info(`index finished.`);
|
||||
|
||||
progressBar.update("Indexing finished.");
|
||||
progressBar.end();
|
||||
}
|
||||
|
||||
|
||||
@ -38,6 +38,8 @@ export async function askCode(message: any, panel: vscode.WebviewPanel|vscode.We
|
||||
_lastMessage = [message];
|
||||
_lastMessage[0]['askCode'] = true;
|
||||
|
||||
const port = await UiUtilWrapper.getLSPBrigePort();
|
||||
|
||||
let pythonVirtualEnv: string|undefined = vscode.workspace.getConfiguration('DevChat').get('PythonVirtualEnv');
|
||||
if (!pythonVirtualEnv) {
|
||||
try {
|
||||
@ -80,34 +82,24 @@ export async function askCode(message: any, panel: vscode.WebviewPanel|vscode.We
|
||||
const workspaceDir = UiUtilWrapper.workspaceFoldersFirstPath();
|
||||
|
||||
try {
|
||||
// create temp directory and file
|
||||
const tempDir = await createTempSubdirectory('devchat/context');
|
||||
const tempFile = path.join(tempDir, "doc_context.txt");
|
||||
|
||||
// If tempFile already exists, delete it
|
||||
if (fs.existsSync(tempFile)) {
|
||||
fs.unlinkSync(tempFile);
|
||||
}
|
||||
|
||||
let outputResult = "";
|
||||
const commandRun = new CommandRun();
|
||||
const command = pythonVirtualEnv.trim();
|
||||
const args = [UiUtilWrapper.extensionPath() + "/tools/askcode_index_query.py", "query", message.text, tempFile];
|
||||
const args = [UiUtilWrapper.extensionPath() + "/tools/askcode_index_query.py", "query", message.text, `${port}`];
|
||||
const result = await commandRun.spawnAsync(command, args, { env: envs, cwd: workspaceDir }, (data) => {
|
||||
logger.channel()?.info(data);
|
||||
outputResult += data;
|
||||
MessageHandler.sendMessage(panel, { command: 'receiveMessagePartial', text: outputResult, hash:"", user:"", isError: false });
|
||||
logger.channel()?.info(data);
|
||||
}, (data) => {
|
||||
logger.channel()?.error(data);
|
||||
}, undefined, undefined);
|
||||
|
||||
// Check if tempFile has been written to
|
||||
if (!fs.existsSync(tempFile) || fs.readFileSync(tempFile, 'utf8') === '') {
|
||||
logger.channel()?.error(`Did not get relevant context from AskCode.`);
|
||||
logger.channel()?.show();
|
||||
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: "Did not get relevant context from AskCode.", hash: "", user: "", date: 0, isError: true });
|
||||
return;
|
||||
}
|
||||
|
||||
// Send message
|
||||
await sendMessage({command: "sendMessage", contextInfo: [{file: tempFile, context: ""}], text: message.text, parent_hash: message.hash}, panel);
|
||||
if (result.exitCode === 0) {
|
||||
MessageHandler.sendMessage(panel, { command: 'receiveMessagePartial', text: result.stdout, hash:"", user:"", isError: false });
|
||||
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: result.stdout, hash:"", user:"", date:0, isError: false });
|
||||
} else {
|
||||
MessageHandler.sendMessage(panel, { command: 'receiveMessage', text: result.stdout + result.stderr, hash: "", user: "", date: 0, isError: true });
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
logger.channel()?.error(`error: ${error.message}`);
|
||||
|
||||
@ -17,6 +17,7 @@ export interface UiUtil {
|
||||
// current selected text
|
||||
selectText(): string | undefined;
|
||||
showErrorMessage(message: string): void;
|
||||
getLSPBrigePort(): Promise<number | undefined>;
|
||||
}
|
||||
|
||||
|
||||
@ -72,5 +73,9 @@ export class UiUtilWrapper {
|
||||
public static showErrorMessage(message: string): void {
|
||||
this._uiUtil?.showErrorMessage(message);
|
||||
}
|
||||
|
||||
public static async getLSPBrigePort(): Promise<number | undefined> {
|
||||
return await this._uiUtil?.getLSPBrigePort();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -122,4 +122,9 @@ export class UiUtilVscode implements UiUtil {
|
||||
public showErrorMessage(message: string): void {
|
||||
vscode.window.showErrorMessage(message);
|
||||
}
|
||||
|
||||
public async getLSPBrigePort(): Promise<number | undefined> {
|
||||
const port = await vscode.commands.executeCommand('LangBrige.getAddress') as number | undefined;;
|
||||
return port;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,210 +1,35 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import tempfile
|
||||
import uuid
|
||||
from chat.ask_codebase.chains.smart_qa import SmartQA
|
||||
|
||||
|
||||
from chat.ask_codebase.store.qdrant import QdrantWrapper as Q, get_client
|
||||
from chat.ask_codebase.indexing.embedding import EmbeddingWrapper as E
|
||||
|
||||
from langchain.embeddings import HuggingFaceEmbeddings
|
||||
from chat.ask_codebase.indexing.loader.file import (
|
||||
FileLoader,
|
||||
FileSource,
|
||||
gen_local_reference_maker,
|
||||
)
|
||||
from chat.util.misc import is_source_code
|
||||
from chat.ask_codebase.chains.simple_qa import SimpleQA
|
||||
from chat.ask_codebase.chains.stuff_dc_qa import StuffDocumentCodeQa
|
||||
|
||||
|
||||
def get_app_data_dir(app_name):
|
||||
home = os.path.expanduser("~")
|
||||
if os.name == "nt": # For Windows
|
||||
appPath = os.path.join(home, "AppData", "Roaming", app_name)
|
||||
else: # For Unix and Linux
|
||||
appPath = os.path.join(home, ".local", "share", app_name)
|
||||
def query(question, lsp_brige_port):
|
||||
root_path = os.getcwd()
|
||||
|
||||
if not os.path.exists(appPath):
|
||||
os.makedirs(appPath)
|
||||
return appPath
|
||||
# Create an instance of SmartQA
|
||||
smart_qa = SmartQA(root_path)
|
||||
|
||||
supportedFileTypes = []
|
||||
# Use SmartQA to get the answer
|
||||
answer = smart_qa.run(question=question, verbose=True, bridge_url=f'http://localhost:{lsp_brige_port}')
|
||||
|
||||
STORAGE_FILE = os.path.join(get_app_data_dir("devchat"), "qdrant_storage2")
|
||||
SOURCE_NAME = ""
|
||||
|
||||
# 为已经分析的文件记录最后修改时间
|
||||
g_file_last_modified_saved = {}
|
||||
|
||||
|
||||
|
||||
def load_file_last_modified(filePath: str):
|
||||
# filePath表示存储了文件最后修改时间的文件名，内容实用JSON存储
|
||||
# 如果文件不存在，表示尚未进行分析，结束函数执行
|
||||
if not os.path.exists(filePath):
|
||||
return {}
|
||||
|
||||
# 如果文件存在，读取文件内容，解析文件中记录的每一个文件的最后修改时间
|
||||
with open(filePath, 'r', encoding="utf-8") as f:
|
||||
fileLastModified = json.load(f)
|
||||
|
||||
return fileLastModified
|
||||
|
||||
|
||||
def save_file_last_modified(filePath: str, fileLastModified: dict):
|
||||
# filePath表示存储了文件最后修改时间的文件名，内容实用JSON存储
|
||||
with open(filePath, 'w+', encoding="utf-8") as f:
|
||||
json.dump(fileLastModified, f)
|
||||
|
||||
return fileLastModified
|
||||
|
||||
def is_source_code_new(filePath: str):
|
||||
# 使用正则表达式来判断一个文件是否是源码文件
|
||||
for pattern in supportedFileTypes:
|
||||
if re.match(pattern.strip(), filePath):
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_file_modified(filePath: str) -> bool:
|
||||
if not is_source_code_new(filePath):
|
||||
return False
|
||||
|
||||
# 获取当前路径
|
||||
currentPath = os.getcwd()
|
||||
# 将filePath转换为相对路径
|
||||
relativePath = os.path.relpath(filePath, currentPath)
|
||||
|
||||
# 检查文件路径中是否包含'.xxx'形式的目录
|
||||
for part in relativePath.split(os.sep):
|
||||
if part.startswith('.') or part in ["node_modules", "__pycache__"]:
|
||||
return False
|
||||
|
||||
# 获取文件上次分析时记录的最后修改时间
|
||||
fileLastModified = g_file_last_modified_saved.get(relativePath, 0)
|
||||
# 获取文件当前的最后修改时间
|
||||
fileCurrentModified = os.path.getmtime(filePath)
|
||||
|
||||
# 如果最后修改时间不同，那么更新记录的最后修改时间，并返回True
|
||||
if fileLastModified != fileCurrentModified:
|
||||
g_file_last_modified_saved[relativePath] = fileCurrentModified
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def index(repo_path: str):
|
||||
try:
|
||||
client = get_client(STORAGE_FILE)
|
||||
source = FileSource(
|
||||
path=repo_path,
|
||||
rel_root=repo_path,
|
||||
ref_maker=gen_local_reference_maker(repo_path),
|
||||
file_filter=is_file_modified,
|
||||
)
|
||||
loader = FileLoader(sources=[source])
|
||||
documents = loader.load()
|
||||
|
||||
e = E(embedding=HuggingFaceEmbeddings())
|
||||
data = e.embed(documents)
|
||||
q = Q.create(
|
||||
source_name=SOURCE_NAME,
|
||||
embedding_cls=HuggingFaceEmbeddings,
|
||||
client=client,
|
||||
)
|
||||
|
||||
q.insert(data)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
import json
|
||||
|
||||
def query(question: str, doc_context: str):
|
||||
try:
|
||||
client = get_client(mode=STORAGE_FILE)
|
||||
q = Q.reuse(
|
||||
source_name=SOURCE_NAME,
|
||||
embedding_cls=HuggingFaceEmbeddings,
|
||||
client=client,
|
||||
)
|
||||
|
||||
chain = StuffDocumentCodeQa(q)
|
||||
|
||||
ans, docs = chain.run(question)
|
||||
|
||||
print(f"\n# Question: \n{question}")
|
||||
print(f"\n# Answer: \n{ans}")
|
||||
print(f"\n# Relevant Documents: \n")
|
||||
doc_dict = {"path": "AskCode Context","content": json.dumps([{"filepath": d.metadata.get('filepath'), "content": d.page_content} for d in docs])}
|
||||
with open(doc_context, 'w') as f:
|
||||
json.dump(doc_dict, f)
|
||||
for d in docs:
|
||||
print(f"- filepath: {d.metadata.get('filepath')}")
|
||||
print(f" location: {d.metadata.get('reference')}\n")
|
||||
|
||||
print(f"Save doc context to {doc_context}")
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit(1)
|
||||
# Print the answer
|
||||
print(answer[0])
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
global supportedFileTypes
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python index_and_query.py [command] [args]")
|
||||
print("Available commands: index, query")
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python index_and_query.py query [question] [port]")
|
||||
sys.exit(1)
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "index":
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python index_and_query.py index [repo_path] [supportedFileTypes]")
|
||||
sys.exit(1)
|
||||
|
||||
repo_path = sys.argv[2]
|
||||
# 获取supportedFileTypes的值
|
||||
supportedFileTypes = sys.argv[3].split(',')
|
||||
index(repo_path)
|
||||
|
||||
elif command == "query":
|
||||
if len(sys.argv) < 4:
|
||||
print("Usage: python index_and_query.py query [question] [doc_context]")
|
||||
sys.exit(1)
|
||||
|
||||
question = sys.argv[2]
|
||||
doc_context = sys.argv[3]
|
||||
query(question, doc_context)
|
||||
|
||||
else:
|
||||
print("Invalid command. Available commands: index, query")
|
||||
sys.exit(1)
|
||||
question = sys.argv[2]
|
||||
port = sys.argv[3]
|
||||
query(question, port)
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
currentPath = os.getcwd()
|
||||
g_file_last_modified_saved = load_file_last_modified('./.chat/.index_modified.json')
|
||||
if os.path.exists(".chat/askcode.json"):
|
||||
with open(".chat/askcode.json", "r") as f:
|
||||
askcode_data = json.load(f)
|
||||
SOURCE_NAME = askcode_data.get("SOURCE_NAME", str(uuid.uuid4()))
|
||||
else:
|
||||
SOURCE_NAME = str(uuid.uuid4())
|
||||
currentPath = os.getcwd()
|
||||
with open(".chat/askcode.json", "w+") as f:
|
||||
json.dump({"SOURCE_NAME": SOURCE_NAME}, f)
|
||||
main()
|
||||
save_file_last_modified('./.chat/.index_modified.json', g_file_last_modified_saved)
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
sys.exit(1)
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user