From fb33deb47a93778741359f231cadae4c63d5b812 Mon Sep 17 00:00:00 2001 From: "bobo.yang" Date: Fri, 3 Nov 2023 11:13:22 +0800 Subject: [PATCH] update to devchat without binary files --- .gitignore | 3 + src/contributes/commandsBase.ts | 52 ++++++++++++++ src/panel/statusBarView.ts | 1 + src/panel/statusBarViewBase.ts | 12 +--- src/toolwrapper/devchat.ts | 48 +++++++++---- src/util/python_installer/app_install.ts | 4 +- src/util/python_installer/install_devchat.ts | 72 +++++++++++++------- 7 files changed, 145 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index a3b371f..58db40b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ node_modules .chat/prompts.db .vscode/settings.json +tools/ + + diff --git a/src/contributes/commandsBase.ts b/src/contributes/commandsBase.ts index bada827..df132e1 100644 --- a/src/contributes/commandsBase.ts +++ b/src/contributes/commandsBase.ts @@ -6,6 +6,58 @@ import { logger } from "../util/logger"; let devchatStatus = ''; + + +function locateCommand(command): string | undefined { + try { + // split lines and choose first line + const binPaths = runCommand(`where ${command}`).toString().trim().split('\n'); + return binPaths[0].trim(); + } catch (error) { + try { + const binPaths = runCommand(`which ${command}`).toString().trim().split('\n'); + return binPaths[0].trim(); + } catch (error) { + return undefined; + } + } +} + +function getDefaultPythonCommand(): string | undefined { + try { + runCommand('python3 -V'); + return locateCommand('python3'); + } catch (error) { + try { + const version = runCommand('python -V'); + if (version.includes('Python 3')) { + return locateCommand('python'); + } + return undefined; + } catch (error) { + return undefined; + } + } +} + +export function getValidPythonCommand(): string | undefined { + try { + const pythonCommand = UiUtilWrapper.getConfiguration('DevChat', 'PythonPath'); + if (pythonCommand) { + return pythonCommand; + } + + const defaultPythonCommand = getDefaultPythonCommand(); + if (defaultPythonCommand) { + UiUtilWrapper.updateConfiguration('DevChat', 'PythonPath', defaultPythonCommand); + } + + return defaultPythonCommand; + } catch (error) { + return undefined; + } +} + export function checkDevChatDependency(showError: boolean = true): boolean { let devChat: string | undefined = UiUtilWrapper.getConfiguration('DevChat', 'DevChatPath'); if (!devChat) { diff --git a/src/panel/statusBarView.ts b/src/panel/statusBarView.ts index fba6943..fbabbce 100644 --- a/src/panel/statusBarView.ts +++ b/src/panel/statusBarView.ts @@ -55,6 +55,7 @@ export function createStatusBarItem(context: vscode.ExtensionContext): vscode.St progressBar.end(); // execute command: DevChat.InstallCommands + vscode.commands.executeCommand('DevChat.InstallCommands'); ExtensionContextHolder.provider?.reloadWebview(); clearInterval(timer); } catch (error) { diff --git a/src/panel/statusBarViewBase.ts b/src/panel/statusBarViewBase.ts index eafc3d5..684341c 100644 --- a/src/panel/statusBarViewBase.ts +++ b/src/panel/statusBarViewBase.ts @@ -43,22 +43,14 @@ export async function dependencyCheck(): Promise<[string, string]> { // define subfunction to check devchat dependency const getDevChatStatus = async (): Promise => { if (devchatStatus === '') { - const bOk = checkDevChatDependency(false); - if (bOk) { - devchatStatus = 'has statisfied the dependency'; - return devchatStatus; - } - devchatStatus = 'installing devchat'; const devchatCommandEnv = await installDevchat(); if (devchatCommandEnv) { - logger.channel()?.info(`devchatCommandEnv: ${devchatCommandEnv}`); - await UiUtilWrapper.updateConfiguration('DevChat', 'DevChatPath', devchatCommandEnv); - + logger.channel()?.info(`Python: ${devchatCommandEnv}`); devchatStatus = 'DevChat has been installed'; return devchatStatus; } else { - logger.channel()?.info(`devchatCommandEnv: undefined`); + logger.channel()?.info(`Python: undefined`); devchatStatus = 'An error occurred during the installation of DevChat'; return devchatStatus; diff --git a/src/toolwrapper/devchat.ts b/src/toolwrapper/devchat.ts index 5d9e1da..905fa6a 100644 --- a/src/toolwrapper/devchat.ts +++ b/src/toolwrapper/devchat.ts @@ -266,6 +266,7 @@ class DevChat { cwd: workspaceDir, env: { PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", ...process.env, OPENAI_API_KEY: activeLlmModelKey, ...openAiApiBaseObject @@ -280,10 +281,11 @@ class DevChat { OPENAI_API_KEY: newActiveLlmModelKey , ...openAiApiBaseObject }; + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; logger.channel()?.info(`Running devchat with arguments: ${args.join(" ")}`); logger.channel()?.info(`Running devchat with environment: ${JSON.stringify(keyInfo)}`); - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnAsyncOptions, onStdoutPartial, undefined, undefined, undefined); + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnAsyncOptions, onStdoutPartial, undefined, undefined, undefined); if (stderr) { let newStderr = stderr; @@ -346,11 +348,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", ...process.env, OPENAI_API_KEY: openaiApiKey, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { @@ -383,11 +388,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", ...process.env, OPENAI_API_KEY: openaiApiKey, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { @@ -420,11 +428,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", ...process.env, OPENAI_API_KEY: openaiApiKey, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { @@ -471,11 +482,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { - ...process.env + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", + ...process.env, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { logger.channel()?.error(`Error: ${stderr}`); @@ -503,11 +517,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { - ...process.env + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", + ...process.env, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { logger.channel()?.error(`Error: ${stderr}`); @@ -526,11 +543,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { - ...process.env + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", + ...process.env, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { logger.channel()?.error(`Error: ${stderr}`); @@ -550,10 +570,14 @@ class DevChat { maxBuffer: 10 * 1024 * 1024, // Set maxBuffer to 10 MB cwd: workspaceDir, env: { - ...process.env + PYTHONUTF8:1, + PYTHONPATH: UiUtilWrapper.extensionPath() + "/tools/site-packages", + ...process.env, }, }; - const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(devChat, args, spawnOptions, undefined, undefined, undefined, undefined); + + const pythonApp = UiUtilWrapper.getConfiguration("DevChat", "PythonPath") || "python3"; + const { exitCode: code, stdout, stderr } = await this.commandRun.spawnAsync(pythonApp, ["-m", "devchat"].concat(args), spawnOptions, undefined, undefined, undefined, undefined); logger.channel()?.info(`Finish devchat with arguments: ${args.join(" ")}`); if (stderr) { diff --git a/src/util/python_installer/app_install.ts b/src/util/python_installer/app_install.ts index 84b67b1..eb5a57a 100644 --- a/src/util/python_installer/app_install.ts +++ b/src/util/python_installer/app_install.ts @@ -13,7 +13,7 @@ import { installPython, installPythonMicromamba } from "./python_install"; // step 3. install devchat in the env -async function createEnvByMamba(pkgName: string, pkgVersion: string, pythonVersion: string) : Promise { +export async function createEnvByMamba(pkgName: string, pkgVersion: string, pythonVersion: string) : Promise { logger.channel()?.info('Find micromamba ...'); const mambaCommand = getMicromambaUrl(); logger.channel()?.info('micromamba url: ' + mambaCommand); @@ -38,7 +38,7 @@ async function createEnvByMamba(pkgName: string, pkgVersion: string, pythonVersi return pythonCommand; } -async function createEnvByConda(pkgName: string, pkgVersion: string, pythonVersion: string) : Promise { +export async function createEnvByConda(pkgName: string, pkgVersion: string, pythonVersion: string) : Promise { // install conda logger.channel()?.info('Install conda ...') const condaCommand = await installConda(); diff --git a/src/util/python_installer/install_devchat.ts b/src/util/python_installer/install_devchat.ts index bc53ce8..cb5b8db 100644 --- a/src/util/python_installer/install_devchat.ts +++ b/src/util/python_installer/install_devchat.ts @@ -3,10 +3,13 @@ */ import { logger } from "../logger"; -import { appInstall } from "./app_install" +import { appInstall, createEnvByConda, createEnvByMamba } from "./app_install"; +import * as os from 'os'; import * as path from 'path'; import * as fs from 'fs'; +import { UiUtilWrapper } from "../uiUtil"; +import { getValidPythonCommand } from "../../contributes/commandsBase"; let isDevChatInstalling: boolean | undefined = undefined; @@ -23,29 +26,52 @@ export function isDevchatInstalling(): boolean { // return: path to devchat, devchat is located in the same directory as python export async function installDevchat(): Promise { try { - logger.channel()?.info(`start installing devchat with python=3.11.4 ...`); - isDevChatInstalling = true; - const pythonCommand = await appInstall('devchat', "", '3.11.4'); - if (!pythonCommand) { - logger.channel()?.error(`failed to install devchat with python=3.11.4`); - logger.channel()?.show(); - isDevChatInstalling = false; - return ''; + // if current os is windows, we don't need to install devchat + if (os.platform() === "win32" && os.arch() === "x64") { + // rewrite ._pth file in python directory + const arch = os.arch(); + const targetPythonPath = os.arch() === "x64"? "python-3.11.6-embed-amd64" : "python-3.11.6-embed-arm64"; + const pythonTargetPath = path.join(UiUtilWrapper.extensionPath(), "tools", targetPythonPath); + const pythonApp = path.join(pythonTargetPath, "python.exe"); + const pythonPathFile = path.join(pythonTargetPath, "python311._pth"); + const sitepackagesPath = path.join(UiUtilWrapper.extensionPath(), "tools", "site-packages"); + + // read content in pythonPathFile + let content = fs.readFileSync(pythonPathFile, { encoding: 'utf-8' }); + // replace %PYTHONPATH% with sitepackagesPath + content = content.replace(/%PYTHONPATH%/g, sitepackagesPath); + // write content to pythonPathFile + fs.writeFileSync(pythonPathFile, content); + + // update DevChat.PythonPath configration + await UiUtilWrapper.updateConfiguration("DevChat", "PythonPath", pythonApp); + return pythonApp; + } else { + // if current os is not windows, we need to get default python path + const pythonPath = getValidPythonCommand(); + if (pythonPath) { + return pythonPath; + } + + logger.channel()?.info(`create env for python ...`); + logger.channel()?.info(`try to create env by mamba ...`); + let pythonCommand = await createEnvByMamba("devchat", "", "3.11.4"); + + if (!pythonCommand || pythonCommand === "") { + logger.channel()?.info(`create env by mamba failed, try to create env by conda ...`); + pythonCommand = await createEnvByConda("devchat", "", "3.11.4"); + } + + if (!pythonCommand) { + logger.channel()?.error('Create env failed'); + logger.channel()?.show(); + return ''; + } + logger.channel()?.info(`Create env success: ${pythonCommand}`); + + await UiUtilWrapper.updateConfiguration("DevChat", "PythonPath", pythonCommand); + return pythonCommand; } - - // Get the directory of pythonCommand - const pythonDirectory = path.dirname(pythonCommand); - - // Get the path of devchat - let devchatPath = path.join(pythonDirectory, 'devchat'); - - // Check if devchatPath exists, if not, try with 'Scripts' subdirectory - if (!fs.existsSync(devchatPath)) { - devchatPath = path.join(pythonDirectory, 'Scripts', 'devchat'); - } - - isDevChatInstalling = false; - return devchatPath; } catch (error) { logger.channel()?.error(`${error}`); logger.channel()?.show();