diff --git a/gui b/gui index 7508749..808bf94 160000 --- a/gui +++ b/gui @@ -1 +1 @@ -Subproject commit 750874981accb3c02d765557f0bd81fa6a2f220b +Subproject commit 808bf945e263cebfdcbbd60696bae84f2a6d594d diff --git a/src/contributes/codecomplete/chunkFilter.ts b/src/contributes/codecomplete/chunkFilter.ts index df9dfca..ca1507d 100644 --- a/src/contributes/codecomplete/chunkFilter.ts +++ b/src/contributes/codecomplete/chunkFilter.ts @@ -277,8 +277,8 @@ export class LLMStreamComplete { // whether lines are repeated some block before async removeRepeatBlock(lines: string[]): Promise< string[] > { - if (lines.length === 0) { - return []; + if (lines.length <= 1) { + return lines; } // find first match line in before 50 lines diff --git a/src/contributes/codecomplete/llm.ts b/src/contributes/codecomplete/llm.ts index 8f8b04b..e54f1f3 100644 --- a/src/contributes/codecomplete/llm.ts +++ b/src/contributes/codecomplete/llm.ts @@ -206,7 +206,7 @@ export async function * devchatComplete(prompt: string) : AsyncGenerator", "<|EOT|>", "", "```", "/", "\n\n"], + stop: ["<|endoftext|>", "<|EOT|>", "", "```", "//", "\n\n"], temperature: 0.2 }; diff --git a/src/contributes/codecomplete/llm/countTokens.ts b/src/contributes/codecomplete/llm/countTokens.ts index 4b84e92..0a39817 100644 --- a/src/contributes/codecomplete/llm/countTokens.ts +++ b/src/contributes/codecomplete/llm/countTokens.ts @@ -42,16 +42,17 @@ function countTokens( // defaults to llama2 because the tokenizer tends to produce more tokens modelName: string = "llama2", ): number { - const encoding = encodingForModel(modelName); - if (Array.isArray(content)) { - return content.reduce((acc, part) => { - return acc + part.type === "imageUrl" - ? countImageTokens(part) - : encoding.encode(part.text ?? "", "all", []).length; - }, 0); - } else { - return encoding.encode(content, "all", []).length; - } + return content.length; + // const encoding = encodingForModel(modelName); + // if (Array.isArray(content)) { + // return content.reduce((acc, part) => { + // return acc + part.type === "imageUrl" + // ? countImageTokens(part) + // : encoding.encode(part.text ?? "", "all", []).length; + // }, 0); + // } else { + // return encoding.encode(content, "all", []).length; + // } } function flattenMessages(msgs: ChatMessage[]): ChatMessage[] { diff --git a/src/contributes/codecomplete/promptCreator.ts b/src/contributes/codecomplete/promptCreator.ts index 048010a..055b330 100644 --- a/src/contributes/codecomplete/promptCreator.ts +++ b/src/contributes/codecomplete/promptCreator.ts @@ -27,7 +27,6 @@ import { findIdentifiersInAstNodeRange } from './ast/findIdentifiers'; import { DevChatConfig } from '../../util/config'; -const CONTEXT_LIMITED_SIZE: number = 6000; const CONTEXT_SIMILAR_LIMITED_SIZE: number = 400; const variableCache: MemoryCacheManager = new MemoryCacheManager(4); @@ -40,7 +39,7 @@ export async function currentFileContext( curColumn: number ) : Promise< { prefix: string, suffix: string } > { const contentTokens = countTokens(contents); - if (contentTokens < CONTEXT_LIMITED_SIZE*0.35) { + if (contentTokens < DevChatConfig.getInstance().get("complete_context_limit", 3000)*0.5) { return curfilePrompt(filepath, contents, curRow, curColumn); } @@ -51,7 +50,7 @@ export async function currentFileContext( const functionRanges = await findFunctionRanges(filepath, ast.rootNode); return await collapseCodeBlock(functionRanges, filepath, contents, curRow, curColumn); - } +} export async function collapseCodeBlock(functions: FunctionRange[], filepath: string, contents: string, curRow: number, curColumn: number) { @@ -186,7 +185,7 @@ async function curfilePrompt(filePath: string, fileContent: string, line: number } const lineTokenCount = countTokens(lineText); - if (prefixTokenCount + lineTokenCount > CONTEXT_LIMITED_SIZE*0.7*0.35) { + if (prefixTokenCount + lineTokenCount > DevChatConfig.getInstance().get("complete_context_limit", 3000)*0.7*0.35) { break; } @@ -195,7 +194,7 @@ async function curfilePrompt(filePath: string, fileContent: string, line: number } // 从光标所在行下一行开始，向下构建后缀 - const suffixMaxToken = CONTEXT_LIMITED_SIZE*0.35 - prefixTokenCount; + const suffixMaxToken = DevChatConfig.getInstance().get("complete_context_limit", 3000)*0.35 - prefixTokenCount; for (let i = line; i < lines.length; i++) { let lineText = lines[i] + '\n'; if (i === line) { @@ -551,17 +550,10 @@ export async function createPrompt(filePath: string, fileContent: string, line: let { prefix, suffix } = await currentFileContext(filePath, fileContent, line, column); - let tokenCount = countTokens(prefix); - - const suffixTokenCount = countTokens(suffix); - if (tokenCount + suffixTokenCount < CONTEXT_LIMITED_SIZE) { - tokenCount += suffixTokenCount; - } else { - suffix = ""; - } + let tokenCount = countTokens(prefix) + countTokens(suffix); let taskDescriptionContextWithCommentPrefix = ""; - if (tokenCount < CONTEXT_LIMITED_SIZE) { + if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { const taskDescriptionContext = await createTaskDescriptionContext(); if (taskDescriptionContext) { taskDescriptionContext.split("\n").forEach(line => { @@ -572,7 +564,7 @@ export async function createPrompt(filePath: string, fileContent: string, line: } const taskDescriptionContextToken = countTokens(taskDescriptionContextWithCommentPrefix); - if (tokenCount + taskDescriptionContextToken < CONTEXT_LIMITED_SIZE) { + if (tokenCount + taskDescriptionContextToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { tokenCount += taskDescriptionContextToken; } else { taskDescriptionContextWithCommentPrefix = ""; @@ -580,9 +572,9 @@ export async function createPrompt(filePath: string, fileContent: string, line: } // let gitDiffContext = GitDiffWatcher.getInstance().getGitDiffResult(); - // if (tokenCount < CONTEXT_LIMITED_SIZE && gitDiffContext.length > 0) { + // if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000) && gitDiffContext.length > 0) { // const gitDiffContextToken = countTokens(gitDiffContext); - // if (tokenCount + gitDiffContextToken < CONTEXT_LIMITED_SIZE) { + // if (tokenCount + gitDiffContextToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { // tokenCount += gitDiffContextToken; // gitDiffContext = "" + gitDiffContext + "\n\n\n\n"; // } else { @@ -592,11 +584,11 @@ export async function createPrompt(filePath: string, fileContent: string, line: let gitDiffContext = ""; let callDefContext = ""; - if (tokenCount < CONTEXT_LIMITED_SIZE) { + if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { const callCodeBlocks = await createContextCallDefine(filePath, fileContent, posoffset); for (const callCodeBlock of callCodeBlocks) { const callBlockToken = countTokens(callCodeBlock.codeblock); - if (tokenCount + callBlockToken > CONTEXT_LIMITED_SIZE) { + if (tokenCount + callBlockToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) { break; } @@ -607,18 +599,18 @@ export async function createPrompt(filePath: string, fileContent: string, line: } let similarBlockContext = ""; - if (tokenCount < CONTEXT_LIMITED_SIZE) { + if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { let similarTokens = 0; const similarContexts: {file: string, text: string}[] = await findSimilarCodeBlock(filePath, fileContent, line, column); for (const similarContext of similarContexts) { const blockToken = countTokens(similarContext.text); - if (tokenCount + blockToken > CONTEXT_LIMITED_SIZE) { + if (tokenCount + blockToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) { continue; } similarTokens += blockToken; - if (similarTokens > CONTEXT_SIMILAR_LIMITED_SIZE) { - continue; + if (similarTokens > CONTEXT_SIMILAR_LIMITED_SIZE) { + continue; } tokenCount += blockToken; @@ -628,11 +620,11 @@ export async function createPrompt(filePath: string, fileContent: string, line: } let symbolContext = ""; - if (tokenCount < CONTEXT_LIMITED_SIZE) { + if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { const symbolDefines: { filepath: string, node: Parser.SyntaxNode, codeblock: string }[] = await symbolDefinesContext(filePath, fileContent, line, column); for (const symbolDefine of symbolDefines ) { const countSymboleToken = countTokens(symbolDefine.codeblock); - if (tokenCount + countSymboleToken > CONTEXT_LIMITED_SIZE) { + if (tokenCount + countSymboleToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) { break; } @@ -644,11 +636,11 @@ export async function createPrompt(filePath: string, fileContent: string, line: } let recentEditContext = ""; - if (tokenCount < CONTEXT_LIMITED_SIZE) { + if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { recentEditContext = await createRecentEditContext(recentEdits, filePath); const countRecentToken = countTokens(recentEditContext); - if (tokenCount + countRecentToken < CONTEXT_LIMITED_SIZE) { + if (tokenCount + countRecentToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { tokenCount += countRecentToken; } else { recentEditContext = ""; @@ -660,7 +652,7 @@ export async function createPrompt(filePath: string, fileContent: string, line: const neighborFiles = await findNeighborFileContext(filePath, fileContent, line, column); if (neighborFiles.length > 0) { const countFileToken = countTokens(neighborFiles[0].text); - if (tokenCount + countFileToken < CONTEXT_LIMITED_SIZE) { + if (tokenCount + countFileToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) { tokenCount += countFileToken; neighborFileContext += `${commentPrefix}neighbor files:\n\n ${neighborFiles[0].file}\n\n`; neighborFileContext += `${neighborFiles[0].text}\n\n\n\n`; diff --git a/src/util/config.ts b/src/util/config.ts index e0b64af..21cb1d0 100644 --- a/src/util/config.ts +++ b/src/util/config.ts @@ -54,7 +54,7 @@ export class DevChatConfig { } } - public get(key: string | string[]): any { + public get(key: string | string[], defaultValue: any = undefined): any { // check if the config file has been modified const currentModifyTime = fs.statSync(this.configFilePath).mtimeMs; if (currentModifyTime > this.lastModifyTime) { @@ -68,7 +68,18 @@ export class DevChatConfig { } else { keys = key; } - return keys.reduce((prev, curr) => prev ? prev[curr] : undefined, this.data); + + let value = this.data; + for (const k of keys) { + if (value && typeof value === 'object' && k in value) { + value = value[k]; + } else { + // If the key is not found or value is not an object, return the default value + return defaultValue || undefined; + } + } + + return value; } public set(key: string | string[], value: any): void {