/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import { Client, PersistentProtocol, ISocket, ProtocolConstants } from 'vs/base/parts/ipc/common/ipc.net'; import { generateUuid } from 'vs/base/common/uuid'; import { RemoteAgentConnectionContext } from 'vs/platform/remote/common/remoteAgentEnvironment'; import { Disposable } from 'vs/base/common/lifecycle'; import { VSBuffer } from 'vs/base/common/buffer'; import { Emitter, Event } from 'vs/base/common/event'; import { RemoteAuthorityResolverError } from 'vs/platform/remote/common/remoteAuthorityResolver'; import { isPromiseCanceledError, onUnexpectedError } from 'vs/base/common/errors'; import { ISignService } from 'vs/platform/sign/common/sign'; import { CancelablePromise, createCancelablePromise } from 'vs/base/common/async'; import { ILogService } from 'vs/platform/log/common/log'; import { IIPCLogger } from 'vs/base/parts/ipc/common/ipc'; const INITIAL_CONNECT_TIMEOUT = 120 * 1000 /* 120s */; const RECONNECT_TIMEOUT = 30 * 1000 /* 30s */; export const enum ConnectionType { Management = 1, ExtensionHost = 2, Tunnel = 3, } function connectionTypeToString(connectionType: ConnectionType): string { switch (connectionType) { case ConnectionType.Management: return 'Management'; case ConnectionType.ExtensionHost: return 'ExtensionHost'; case ConnectionType.Tunnel: return 'Tunnel'; } } export interface AuthRequest { type: 'auth'; auth: string; } export interface SignRequest { type: 'sign'; data: string; } export interface ConnectionTypeRequest { type: 'connectionType'; commit?: string; signedData?: string; desiredConnectionType?: ConnectionType; args?: any; } export interface ErrorMessage { type: 'error'; reason: string; } export interface OKMessage { type: 'ok'; } export type HandshakeMessage = AuthRequest | SignRequest | ConnectionTypeRequest | ErrorMessage | OKMessage; interface ISimpleConnectionOptions { commit: string | undefined; host: string; port: number; connectionToken: string | undefined; reconnectionToken: string; reconnectionProtocol: PersistentProtocol | null; socketFactory: ISocketFactory; signService: ISignService; logService: ILogService; } export interface IConnectCallback { (err: any | undefined, socket: ISocket | undefined): void; } export interface ISocketFactory { connect(host: string, port: number, query: string, callback: IConnectCallback): void; } async function readOneControlMessage(protocol: PersistentProtocol): Promise { const raw = await Event.toPromise(protocol.onControlMessage); const msg = JSON.parse(raw.toString()); const error = getErrorFromMessage(msg); if (error) { throw error; } return msg; } function waitWithTimeout(promise: Promise, timeout: number): Promise { return new Promise((resolve, reject) => { const timeoutToken = setTimeout(() => { const error: any = new Error('Timeout'); error.code = 'ETIMEDOUT'; error.syscall = 'connect'; reject(error); }, timeout); promise.then( (result) => { clearTimeout(timeoutToken); resolve(result); }, (error) => { clearTimeout(timeoutToken); reject(error); } ); }); } function createSocket(socketFactory: ISocketFactory, host: string, port: number, query: string): Promise { return new Promise((resolve, reject) => { socketFactory.connect(host, port, query, (err: any, socket: ISocket | undefined) => { if (err || !socket) { return reject(err); } resolve(socket); }); }); } async function connectToRemoteExtensionHostAgent(options: ISimpleConnectionOptions, connectionType: ConnectionType, args: any | undefined): Promise<{ protocol: PersistentProtocol; ownsProtocol: boolean; }> { const logPrefix = connectLogPrefix(options, connectionType); options.logService.trace(`${logPrefix} 1/6. invoking socketFactory.connect().`); let socket: ISocket; try { socket = await createSocket(options.socketFactory, options.host, options.port, `reconnectionToken=${options.reconnectionToken}&reconnection=${options.reconnectionProtocol ? 'true' : 'false'}`); } catch (error) { options.logService.error(`${logPrefix} socketFactory.connect() failed. Error:`); options.logService.error(error); throw error; } options.logService.trace(`${logPrefix} 2/6. socketFactory.connect() was successful.`); let protocol: PersistentProtocol; let ownsProtocol: boolean; if (options.reconnectionProtocol) { options.reconnectionProtocol.beginAcceptReconnection(socket, null); protocol = options.reconnectionProtocol; ownsProtocol = false; } else { protocol = new PersistentProtocol(socket, null); ownsProtocol = true; } options.logService.trace(`${logPrefix} 3/6. sending AuthRequest control message.`); const authRequest: AuthRequest = { type: 'auth', auth: options.connectionToken || '00000000000000000000' }; protocol.sendControl(VSBuffer.fromString(JSON.stringify(authRequest))); try { const msg = await waitWithTimeout(readOneControlMessage(protocol), 10000); if (msg.type !== 'sign' || typeof msg.data !== 'string') { const error: any = new Error('Unexpected handshake message'); error.code = 'VSCODE_CONNECTION_ERROR'; throw error; } options.logService.trace(`${logPrefix} 4/6. received SignRequest control message.`); const signed = await options.signService.sign(msg.data); const connTypeRequest: ConnectionTypeRequest = { type: 'connectionType', commit: options.commit, signedData: signed, desiredConnectionType: connectionType }; if (args) { connTypeRequest.args = args; } options.logService.trace(`${logPrefix} 5/6. sending ConnectionTypeRequest control message.`); protocol.sendControl(VSBuffer.fromString(JSON.stringify(connTypeRequest))); return { protocol, ownsProtocol }; } catch (error) { if (error && error.code === 'ETIMEDOUT') { options.logService.error(`${logPrefix} the handshake took longer than 10 seconds. Error:`); options.logService.error(error); } if (error && error.code === 'VSCODE_CONNECTION_ERROR') { options.logService.error(`${logPrefix} received error control message when negotiating connection. Error:`); options.logService.error(error); } if (ownsProtocol) { safeDisposeProtocolAndSocket(protocol); } throw error; } } interface IManagementConnectionResult { protocol: PersistentProtocol; } async function connectToRemoteExtensionHostAgentAndReadOneMessage(options: ISimpleConnectionOptions, connectionType: ConnectionType, args: any | undefined): Promise<{ protocol: PersistentProtocol; firstMessage: any }> { const startTime = Date.now(); const logPrefix = connectLogPrefix(options, connectionType); const { protocol, ownsProtocol } = await connectToRemoteExtensionHostAgent(options, connectionType, args); return new Promise<{ protocol: PersistentProtocol; firstMessage: any }>((c, e) => { const registration = protocol.onControlMessage(raw => { registration.dispose(); const msg = JSON.parse(raw.toString()); const error = getErrorFromMessage(msg); if (error) { options.logService.error(`${logPrefix} received error control message when negotiating connection. Error:`); options.logService.error(error); if (ownsProtocol) { safeDisposeProtocolAndSocket(protocol); } return e(error); } if (options.reconnectionProtocol) { options.reconnectionProtocol.endAcceptReconnection(); } options.logService.trace(`${logPrefix} 6/6. handshake finished, connection is up and running after ${logElapsed(startTime)}!`); c({ protocol, firstMessage: msg }); }); }); } async function doConnectRemoteAgentManagement(options: ISimpleConnectionOptions): Promise { const { protocol } = await connectToRemoteExtensionHostAgentAndReadOneMessage(options, ConnectionType.Management, undefined); return { protocol }; } export interface IRemoteExtensionHostStartParams { language: string; debugId?: string; break?: boolean; port?: number | null; env?: { [key: string]: string | null }; } interface IExtensionHostConnectionResult { protocol: PersistentProtocol; debugPort?: number; } async function doConnectRemoteAgentExtensionHost(options: ISimpleConnectionOptions, startArguments: IRemoteExtensionHostStartParams): Promise { const { protocol, firstMessage } = await connectToRemoteExtensionHostAgentAndReadOneMessage(options, ConnectionType.ExtensionHost, startArguments); const debugPort = firstMessage && firstMessage.debugPort; return { protocol, debugPort }; } export interface ITunnelConnectionStartParams { port: number; } async function doConnectRemoteAgentTunnel(options: ISimpleConnectionOptions, startParams: ITunnelConnectionStartParams): Promise { const startTime = Date.now(); const logPrefix = connectLogPrefix(options, ConnectionType.Tunnel); const { protocol } = await connectToRemoteExtensionHostAgent(options, ConnectionType.Tunnel, startParams); options.logService.trace(`${logPrefix} 6/6. handshake finished, connection is up and running after ${logElapsed(startTime)}!`); return protocol; } export interface IConnectionOptions { commit: string | undefined; socketFactory: ISocketFactory; addressProvider: IAddressProvider; signService: ISignService; logService: ILogService; ipcLogger: IIPCLogger | null; } async function resolveConnectionOptions(options: IConnectionOptions, reconnectionToken: string, reconnectionProtocol: PersistentProtocol | null): Promise { const { host, port, connectionToken } = await options.addressProvider.getAddress(); return { commit: options.commit, host: host, port: port, connectionToken: connectionToken, reconnectionToken: reconnectionToken, reconnectionProtocol: reconnectionProtocol, socketFactory: options.socketFactory, signService: options.signService, logService: options.logService }; } export interface IAddress { host: string; port: number; connectionToken: string | undefined; } export interface IAddressProvider { getAddress(): Promise; } export async function connectRemoteAgentManagement(options: IConnectionOptions, remoteAuthority: string, clientId: string): Promise { try { const reconnectionToken = generateUuid(); const simpleOptions = await resolveConnectionOptions(options, reconnectionToken, null); const { protocol } = await connectWithTimeLimit(simpleOptions.logService, doConnectRemoteAgentManagement(simpleOptions), INITIAL_CONNECT_TIMEOUT); return new ManagementPersistentConnection(options, remoteAuthority, clientId, reconnectionToken, protocol); } catch (err) { options.logService.error(`[remote-connection] An error occurred in the very first connect attempt, it will be treated as a permanent error! Error:`); options.logService.error(err); PersistentConnection.triggerPermanentFailure(RemoteAuthorityResolverError.isHandled(err)); throw err; } } export async function connectRemoteAgentExtensionHost(options: IConnectionOptions, startArguments: IRemoteExtensionHostStartParams): Promise { try { const reconnectionToken = generateUuid(); const simpleOptions = await resolveConnectionOptions(options, reconnectionToken, null); const { protocol, debugPort } = await connectWithTimeLimit(simpleOptions.logService, doConnectRemoteAgentExtensionHost(simpleOptions, startArguments), INITIAL_CONNECT_TIMEOUT); return new ExtensionHostPersistentConnection(options, startArguments, reconnectionToken, protocol, debugPort); } catch (err) { options.logService.error(`[remote-connection] An error occurred in the very first connect attempt, it will be treated as a permanent error! Error:`); options.logService.error(err); PersistentConnection.triggerPermanentFailure(RemoteAuthorityResolverError.isHandled(err)); throw err; } } export async function connectRemoteAgentTunnel(options: IConnectionOptions, tunnelRemotePort: number): Promise { const simpleOptions = await resolveConnectionOptions(options, generateUuid(), null); const protocol = await connectWithTimeLimit(simpleOptions.logService, doConnectRemoteAgentTunnel(simpleOptions, { port: tunnelRemotePort }), INITIAL_CONNECT_TIMEOUT); return protocol; } function sleep(seconds: number): CancelablePromise { return createCancelablePromise(token => { return new Promise((resolve, reject) => { const timeout = setTimeout(resolve, seconds * 1000); token.onCancellationRequested(() => { clearTimeout(timeout); resolve(); }); }); }); } export const enum PersistentConnectionEventType { ConnectionLost, ReconnectionWait, ReconnectionRunning, ReconnectionPermanentFailure, ConnectionGain } export class ConnectionLostEvent { public readonly type = PersistentConnectionEventType.ConnectionLost; constructor( public readonly millisSinceLastIncomingData: number ) { } } export class ReconnectionWaitEvent { public readonly type = PersistentConnectionEventType.ReconnectionWait; constructor( public readonly durationSeconds: number, public readonly millisSinceLastIncomingData: number, private readonly cancellableTimer: CancelablePromise ) { } public skipWait(): void { this.cancellableTimer.cancel(); } } export class ReconnectionRunningEvent { public readonly type = PersistentConnectionEventType.ReconnectionRunning; constructor( public readonly millisSinceLastIncomingData: number ) { } } export class ConnectionGainEvent { public readonly type = PersistentConnectionEventType.ConnectionGain; } export class ReconnectionPermanentFailureEvent { public readonly type = PersistentConnectionEventType.ReconnectionPermanentFailure; constructor( public readonly handled: boolean ) { } } export type PersistentConnectionEvent = ConnectionGainEvent | ConnectionLostEvent | ReconnectionWaitEvent | ReconnectionRunningEvent | ReconnectionPermanentFailureEvent; abstract class PersistentConnection extends Disposable { public static triggerPermanentFailure(handled: boolean): void { this._permanentFailure = true; this._permanentFailureHandled = handled; this._instances.forEach(instance => instance._gotoPermanentFailure(handled)); } private static _permanentFailure: boolean = false; private static _permanentFailureHandled: boolean = false; private static _instances: PersistentConnection[] = []; private readonly _onDidStateChange = this._register(new Emitter()); public readonly onDidStateChange = this._onDidStateChange.event; protected readonly _options: IConnectionOptions; public readonly reconnectionToken: string; public readonly protocol: PersistentProtocol; private _isReconnecting: boolean; constructor(private readonly _connectionType: ConnectionType, options: IConnectionOptions, reconnectionToken: string, protocol: PersistentProtocol) { super(); this._options = options; this.reconnectionToken = reconnectionToken; this.protocol = protocol; this._isReconnecting = false; this._onDidStateChange.fire(new ConnectionGainEvent()); this._register(protocol.onSocketClose(() => this._beginReconnecting())); this._register(protocol.onSocketTimeout(() => this._beginReconnecting())); PersistentConnection._instances.push(this); if (PersistentConnection._permanentFailure) { this._gotoPermanentFailure(PersistentConnection._permanentFailureHandled); } } private async _beginReconnecting(): Promise { // Only have one reconnection loop active at a time. if (this._isReconnecting) { return; } try { this._isReconnecting = true; await this._runReconnectingLoop(); } finally { this._isReconnecting = false; } } private async _runReconnectingLoop(): Promise { if (PersistentConnection._permanentFailure) { // no more attempts! return; } const logPrefix = commonLogPrefix(this._connectionType, this.reconnectionToken, true); this._options.logService.info(`${logPrefix} starting reconnecting loop. You can get more information with the trace log level.`); this._onDidStateChange.fire(new ConnectionLostEvent(this.protocol.getMillisSinceLastIncomingData())); const TIMES = [0, 5, 5, 10, 10, 10, 10, 10, 30]; const disconnectStartTime = Date.now(); let attempt = -1; do { attempt++; const waitTime = (attempt < TIMES.length ? TIMES[attempt] : TIMES[TIMES.length - 1]); try { if (waitTime > 0) { const sleepPromise = sleep(waitTime); this._onDidStateChange.fire(new ReconnectionWaitEvent(waitTime, this.protocol.getMillisSinceLastIncomingData(), sleepPromise)); this._options.logService.info(`${logPrefix} waiting for ${waitTime} seconds before reconnecting...`); try { await sleepPromise; } catch { } // User canceled timer } if (PersistentConnection._permanentFailure) { this._options.logService.error(`${logPrefix} permanent failure occurred while running the reconnecting loop.`); break; } // connection was lost, let's try to re-establish it this._onDidStateChange.fire(new ReconnectionRunningEvent(this.protocol.getMillisSinceLastIncomingData())); this._options.logService.info(`${logPrefix} resolving connection...`); const simpleOptions = await resolveConnectionOptions(this._options, this.reconnectionToken, this.protocol); this._options.logService.info(`${logPrefix} connecting to ${simpleOptions.host}:${simpleOptions.port}...`); await connectWithTimeLimit(simpleOptions.logService, this._reconnect(simpleOptions), RECONNECT_TIMEOUT); this._options.logService.info(`${logPrefix} reconnected!`); this._onDidStateChange.fire(new ConnectionGainEvent()); break; } catch (err) { if (err.code === 'VSCODE_CONNECTION_ERROR') { this._options.logService.error(`${logPrefix} A permanent error occurred in the reconnecting loop! Will give up now! Error:`); this._options.logService.error(err); PersistentConnection.triggerPermanentFailure(false); break; } if (Date.now() - disconnectStartTime > ProtocolConstants.ReconnectionGraceTime) { this._options.logService.error(`${logPrefix} An error occurred while reconnecting, but it will be treated as a permanent error because the reconnection grace time has expired! Will give up now! Error:`); this._options.logService.error(err); PersistentConnection.triggerPermanentFailure(false); break; } if (RemoteAuthorityResolverError.isTemporarilyNotAvailable(err)) { this._options.logService.info(`${logPrefix} A temporarily not available error occurred while trying to reconnect, will try again...`); this._options.logService.trace(err); // try again! continue; } if ((err.code === 'ETIMEDOUT' || err.code === 'ENETUNREACH' || err.code === 'ECONNREFUSED' || err.code === 'ECONNRESET') && err.syscall === 'connect') { this._options.logService.info(`${logPrefix} A network error occurred while trying to reconnect, will try again...`); this._options.logService.trace(err); // try again! continue; } if (isPromiseCanceledError(err)) { this._options.logService.info(`${logPrefix} A promise cancelation error occurred while trying to reconnect, will try again...`); this._options.logService.trace(err); // try again! continue; } if (err instanceof RemoteAuthorityResolverError) { this._options.logService.error(`${logPrefix} A RemoteAuthorityResolverError occurred while trying to reconnect. Will give up now! Error:`); this._options.logService.error(err); PersistentConnection.triggerPermanentFailure(RemoteAuthorityResolverError.isHandled(err)); break; } this._options.logService.error(`${logPrefix} An unknown error occurred while trying to reconnect, since this is an unknown case, it will be treated as a permanent error! Will give up now! Error:`); this._options.logService.error(err); PersistentConnection.triggerPermanentFailure(false); break; } } while (!PersistentConnection._permanentFailure); } private _gotoPermanentFailure(handled: boolean): void { this._onDidStateChange.fire(new ReconnectionPermanentFailureEvent(handled)); safeDisposeProtocolAndSocket(this.protocol); } protected abstract _reconnect(options: ISimpleConnectionOptions): Promise; } export class ManagementPersistentConnection extends PersistentConnection { public readonly client: Client; constructor(options: IConnectionOptions, remoteAuthority: string, clientId: string, reconnectionToken: string, protocol: PersistentProtocol) { super(ConnectionType.Management, options, reconnectionToken, protocol); this.client = this._register(new Client(protocol, { remoteAuthority: remoteAuthority, clientId: clientId }, options.ipcLogger)); } protected async _reconnect(options: ISimpleConnectionOptions): Promise { await doConnectRemoteAgentManagement(options); } } export class ExtensionHostPersistentConnection extends PersistentConnection { private readonly _startArguments: IRemoteExtensionHostStartParams; public readonly debugPort: number | undefined; constructor(options: IConnectionOptions, startArguments: IRemoteExtensionHostStartParams, reconnectionToken: string, protocol: PersistentProtocol, debugPort: number | undefined) { super(ConnectionType.ExtensionHost, options, reconnectionToken, protocol); this._startArguments = startArguments; this.debugPort = debugPort; } protected async _reconnect(options: ISimpleConnectionOptions): Promise { await doConnectRemoteAgentExtensionHost(options, this._startArguments); } } function connectWithTimeLimit(logService: ILogService, p: Promise, timeLimit: number): Promise { return new Promise((resolve, reject) => { let timeout = setTimeout(() => { const err: any = new Error('Time limit reached'); err.code = 'ETIMEDOUT'; err.syscall = 'connect'; logService.error(`[remote-connection] The time limit has been reached for a connection. Error:`); logService.error(err); reject(err); }, timeLimit); p.then((value) => { clearTimeout(timeout); resolve(value); }, (err) => { clearTimeout(timeout); reject(err); }); }); } function safeDisposeProtocolAndSocket(protocol: PersistentProtocol): void { try { protocol.acceptDisconnect(); const socket = protocol.getSocket(); protocol.dispose(); socket.dispose(); } catch (err) { onUnexpectedError(err); } } function getErrorFromMessage(msg: any): Error | null { if (msg && msg.type === 'error') { const error = new Error(`Connection error: ${msg.reason}`); (error).code = 'VSCODE_CONNECTION_ERROR'; return error; } return null; } function stringRightPad(str: string, len: number): string { while (str.length < len) { str += ' '; } return str; } function commonLogPrefix(connectionType: ConnectionType, reconnectionToken: string, isReconnect: boolean): string { return `[remote-connection][${stringRightPad(connectionTypeToString(connectionType), 13)}][${reconnectionToken.substr(0, 5)}…][${isReconnect ? 'reconnect' : 'initial'}]`; } function connectLogPrefix(options: ISimpleConnectionOptions, connectionType: ConnectionType): string { return `${commonLogPrefix(connectionType, options.reconnectionToken, !!options.reconnectionProtocol)}[${options.host}:${options.port}]`; } function logElapsed(startTime: number): string { return `${Date.now() - startTime} ms`; }