diff --git a/api/designManager.js b/api/designManager.js index dceadc4..68c2597 100644 --- a/api/designManager.js +++ b/api/designManager.js @@ -43,9 +43,37 @@ async function fetchRawAssetByPath(accountId, path) { }); } +async function fetchModulesByPath(accountId, path) { + return http.get(accountId, { + uri: `${DESIGN_MANAGER_API_PATH}/modules/by-path/${path}?portalId=${accountId}`, + }); +} + +async function fetchPreviewModules(accountId, token) { + return http.get(accountId, { + uri: `${DESIGN_MANAGER_API_PATH}/modules/local-preview?portalId=${accountId}&previewToken=${token}`, + }); +} + +async function fetchTemplatesByPath(accountId, path) { + return http.get(accountId, { + uri: `${DESIGN_MANAGER_API_PATH}/templates/by-path/${path}?portalId=${accountId}`, + }); +} + +async function fetchPreviewTemplates(accountId, token) { + return http.get(accountId, { + uri: `${DESIGN_MANAGER_API_PATH}/templates/local-preview?portalId=${accountId}&previewToken=${token}`, + }); +} + module.exports = { fetchBuiltinMapping, fetchMenus, fetchRawAssetByPath, fetchThemes, + fetchModulesByPath, + fetchPreviewModules, + fetchTemplatesByPath, + fetchPreviewTemplates, }; diff --git a/api/domains.js b/api/domains.js new file mode 100644 index 0000000..18d7675 --- /dev/null +++ b/api/domains.js @@ -0,0 +1,16 @@ +const http = require('../http'); + +const DOMAINS_API_PATH = `/cms/v3/domains`; + +async function fetchDomains(accountId) { + const result = await http.get(accountId, { + uri: DOMAINS_API_PATH, + json: true, + }); + + return result.results; +} + +module.exports = { + fetchDomains, +}; diff --git a/api/preview.js b/api/preview.js new file mode 100644 index 0000000..cbc3d9b --- /dev/null +++ b/api/preview.js @@ -0,0 +1,16 @@ +const { request } = require('../http'); + +async function fetchPreviewRender(url, sessionInfo) { + const { sessionToken } = sessionInfo; + + const urlObject = new URL(url); + + urlObject.searchParams.append('localPreviewToken', sessionToken); + urlObject.searchParams.append('hsCacheBuster', Date.now()); + + return request(urlObject.href); +} + +module.exports = { + fetchPreviewRender, +}; diff --git a/lang/en.lyaml b/lang/en.lyaml index 9051199..d1b65eb 100644 --- a/lang/en.lyaml +++ b/lang/en.lyaml @@ -52,5 +52,3 @@ en: fieldsJsSyntaxError: "There was an error converting JS file \"{{ path }}\"" fieldsJsNotReturnArray: "There was an error loading JS file \"{{ path }}\". Expected type \"Array\" but received type \"{{ returned }}\" . Make sure that your function returns an array" fieldsJsNotFunction: "There was an error loading JS file \"{{ path }}\". Expected type \"Function\" but received type \"{{ returned }}\". Make sure that your default export is a function." - - diff --git a/lib/preview.js b/lib/preview.js new file mode 100644 index 0000000..f4b264f --- /dev/null +++ b/lib/preview.js @@ -0,0 +1,257 @@ +const http = require('http'); +const path = require('path'); +const chokidar = require('chokidar'); +const express = require('express'); +const { v4: uuidv4 } = require('uuid'); +const { logger } = require('../logger'); +const { + ApiErrorContext, + logApiErrorInstance, + logApiUploadErrorInstance, +} = require('../errorHandlers'); +const { uploadFolder } = require('./uploadFolder'); +const { shouldIgnoreFile, ignoreFile } = require('../ignoreRules'); +const { getFileMapperQueryValues } = require('../fileMapper'); +const { upload, deleteFile } = require('../api/fileMapper'); +const escapeRegExp = require('./escapeRegExp'); +const { convertToUnixPath, isAllowedExtension } = require('../path'); +const { triggerNotify } = require('./notify'); +const { getAccountConfig } = require('./config'); +const { createPreviewServerRoutes } = require('./preview/createRoutes'); +const { + getPortalDomains, + isUngatedForPreview, +} = require('./preview/previewUtils'); +const { markRemoteFsDirty } = require('./preview/routes/meta'); +const { startSprocketMenuServer } = require('./preview/sprocketMenuServer'); +const { + createHttpsRedirectingServer, +} = require('./preview/httpsRedirectingServer'); + +const fileMapperArgs = getFileMapperQueryValues({ + mode: 'publish', +}); + +async function uploadFile(accountId, src, dest) { + logger.debug(`Attempting to upload file "${src}" to "${dest}"`); + + try { + await upload(accountId, src, dest, fileMapperArgs); + logger.log(`Uploaded file ${src} to ${dest}`); + markRemoteFsDirty(); + } catch { + const uploadFailureMessage = `Uploading file ${src} to ${dest} failed`; + logger.debug(uploadFailureMessage); + logger.debug(`Retrying to upload file "${src}" to "${dest}"`); + try { + await upload(accountId, src, dest, fileMapperArgs); + markRemoteFsDirty(); + } catch (error) { + logger.error(uploadFailureMessage); + logApiUploadErrorInstance( + error, + new ApiErrorContext({ + accountId, + request: dest, + payload: src, + }) + ); + } + } +} + +async function deleteRemoteFile(accountId, remoteFilePath) { + logger.debug(`Attempting to delete file "${remoteFilePath}"`); + + try { + await deleteFile(accountId, remoteFilePath, fileMapperArgs); + logger.log(`Deleted file ${remoteFilePath}`); + markRemoteFsDirty(); + } catch (error) { + logger.error(`Deleting file ${remoteFilePath} failed`); + logger.debug(`Retrying deletion of file ${remoteFilePath}`); + try { + await deleteFile(accountId, remoteFilePath, fileMapperArgs); + markRemoteFsDirty(); + } catch (error) { + logger.error(`Deleting file ${remoteFilePath} failed`); + logApiErrorInstance( + error, + new ApiErrorContext({ + accountId, + request: remoteFilePath, + }) + ); + } + } +} + +const getDesignManagerPath = (src, dest, file) => { + const regex = new RegExp(`^${escapeRegExp(src)}`); + const relativePath = file.replace(regex, ''); + return convertToUnixPath(path.join(dest, relativePath)); +}; + +const buildDeleteFileFromPreviewBufferCallback = (sessionInfo, type) => { + const { accountId, src, dest, notify } = sessionInfo; + + return filePath => { + if (shouldIgnoreFile(filePath)) { + logger.debug(`Skipping ${filePath} due to an ignore rule`); + return; + } + + const remotePath = getDesignManagerPath(src, dest, filePath); + const deletePromise = deleteRemoteFile(accountId, remotePath); + triggerNotify(notify, 'Removed', filePath, deletePromise); + }; +}; + +const buildUploadFileToPreviewBufferCallback = (sessionInfo, notifyMessage) => { + const { accountId, src, dest, notify } = sessionInfo; + + return async filePath => { + if (!isAllowedExtension(filePath)) { + logger.debug(`Skipping ${filePath} due to unsupported extension`); + return; + } + if (shouldIgnoreFile(filePath)) { + logger.debug(`Skipping ${filePath} due to an ignore rule`); + return; + } + const destPath = getDesignManagerPath(src, dest, filePath); + const uploadPromise = uploadFile(accountId, filePath, destPath); + triggerNotify(notify, notifyMessage, filePath, uploadPromise); + }; +}; + +const initialPreviewBufferUpload = async (sessionInfo, filePaths, uploadOptions) => { + const { accountId, src, dest } = sessionInfo; + const { onFinishCallback, ...rest } = uploadOptions; + + const results = await uploadFolder(accountId, src, dest, fileMapperArgs, rest, filePaths); + onFinishCallback(results); +}; + +const startPreviewWatcher = async sessionInfo => { + const { src } = sessionInfo; + let watcherIsReady = false; + + const watcher = chokidar.watch(src, { + ignoreInitial: true, // makes initial addition of files not trigger the watcher + ignored: file => shouldIgnoreFile(file), + }); + + const addFileCallback = buildUploadFileToPreviewBufferCallback( + sessionInfo, + 'Added' + ); + const changeFileCallback = buildUploadFileToPreviewBufferCallback( + sessionInfo, + 'Change' + ); + const deleteFileCallback = buildDeleteFileFromPreviewBufferCallback( + sessionInfo, + 'file' + ); + const deleteFolderCallback = buildDeleteFileFromPreviewBufferCallback( + sessionInfo, + 'folder' + ); + + watcher.on('ready', () => { + watcherIsReady = true; + }); + watcher.on('add', addFileCallback); + watcher.on('change', changeFileCallback); + watcher.on('error', error => + logger.error(`An error occurred while watching files: ${error}`) + ); + + watcher.on('unlink', deleteFileCallback); + watcher.on('unlinkDir', deleteFolderCallback); + + function sleep(ms) { + return new Promise(resolve => { + setTimeout(resolve, ms); + }); + } + while (!watcherIsReady) { + await sleep(1); + // do nothing... surely there's a better way to do this... + } + return watcher; +}; + +const createLocalHttpServer = async sessionInfo => { + const expressServer = express(); + expressServer.use('/', await createPreviewServerRoutes(sessionInfo)); + + return expressServer; +}; + +const preview = async ( + accountId, + src, + dest, + { notify, filePaths, skipUpload, noSsl, port, uploadOptions } +) => { + const accountConfig = getAccountConfig(accountId); + const domains = await getPortalDomains(accountId); + const sessionToken = uuidv4(); + const PORT = port || 3000; + const protocol = noSsl ? 'http' : 'https'; + + const sessionInfo = { + src, + dest: `@preview/${sessionToken}/${dest}`, + fakeDest: dest, + portalName: accountConfig.name, + accountId, + env: accountConfig.env, + personalAccessKey: accountConfig.personalAccessKey, + // we find hublet later in the content metadata fetch + // can we get that ahead of time? hardcoding it for now + hublet: 'na1', + sessionToken, + domains, + PORT, + protocol, + }; + const ungated = await isUngatedForPreview(sessionInfo); + if (!ungated) { + logger.log( + `Portal ${accountId} is missing a required gate for this feature.` + ); + process.exit(); + } + if (notify) { + ignoreFile(notify); + } + + if (!skipUpload) { + await initialPreviewBufferUpload(sessionInfo, filePaths, uploadOptions); + } + const expressServer = await createLocalHttpServer(sessionInfo); + const previewWatcher = await startPreviewWatcher(sessionInfo); + + if (!noSsl) { + const { + server, + innerHTTPServer, + innerHTTPSServer, + } = await createHttpsRedirectingServer(expressServer, domains); + server.listen(PORT); + } else { + const httpServer = http.createServer(expressServer); + httpServer.listen(PORT); + } + startSprocketMenuServer(sessionInfo); + logger.log( + `Local dev server started at ${protocol}://hslocal.net:${PORT} for portal ${accountId}` + ); +}; + +module.exports = { + preview, +}; diff --git a/lib/preview/createRoutes.js b/lib/preview/createRoutes.js new file mode 100644 index 0000000..ba743c7 --- /dev/null +++ b/lib/preview/createRoutes.js @@ -0,0 +1,38 @@ +const { Router } = require('express'); +const { logger } = require('./../../logger'); + +const { buildIndexRouteHandler } = require('./routes/index.js'); +const { buildModuleRouteHandler } = require('./routes/module.js'); +const { buildTemplateRouteHandler } = require('./routes/template.js'); +const { buildMetaRouteHandler } = require('./routes/meta.js'); + +const { buildProxyRouteHandler } = require('./routes/proxyPathPageRouteHandler.js'); +const { buildProxyPageRouteHandler } = require('./routes/proxyPageRouteHandler.js'); +const { proxyPathPageResourceRedirect } = require('./routes/proxyPathPageResourceRedirect.js'); +const { proxyPageResourceRedirect } = require('./routes/proxyPageResourceRedirect.js'); + +const createPreviewServerRoutes = async (sessionInfo) => { + const previewServerRouter = Router(); + previewServerRouter.get('/proxy', buildProxyRouteHandler(sessionInfo)); + previewServerRouter.get('/module/:modulePath(*)', buildModuleRouteHandler(sessionInfo)); + previewServerRouter.get('/template/:templatePath(*)', buildTemplateRouteHandler(sessionInfo)); + // fetches server metadata from the client (used by refresh script to check if fs has been changed) + previewServerRouter.get('/meta', buildMetaRouteHandler(sessionInfo)); + // handles resources on the proxied page, so a fetch from relative path gets proxied too + previewServerRouter.get('/*', proxyPathPageResourceRedirect) + previewServerRouter.get('/*', proxyPageResourceRedirect); + previewServerRouter.post('/*', proxyPageResourceRedirect); + previewServerRouter.delete('/*', proxyPageResourceRedirect); + previewServerRouter.head('/*', proxyPageResourceRedirect); + previewServerRouter.put('/*', proxyPageResourceRedirect); + previewServerRouter.options('/*', proxyPageResourceRedirect); + previewServerRouter.get('/*', buildProxyPageRouteHandler(sessionInfo)); + // index route + previewServerRouter.get('/', buildIndexRouteHandler(sessionInfo)); + + return previewServerRouter; +} + +module.exports = { + createPreviewServerRoutes, +} diff --git a/lib/preview/httpsRedirectingServer.js b/lib/preview/httpsRedirectingServer.js new file mode 100644 index 0000000..604530c --- /dev/null +++ b/lib/preview/httpsRedirectingServer.js @@ -0,0 +1,86 @@ +const http = require('http'); +const https = require('https'); +const net = require('net'); +const os = require('os') +const { unlinkSync } = require('fs'); +const { silenceConsoleWhile } = require('./previewUtils'); + +const createCert = async (domainsToProxy) => { + const additionalMkcertHosts = domainsToProxy + .map(proxyDomain => [ + `${proxyDomain.domain}.localhost`, + `${proxyDomain.domain}.hslocal.net` + ]) + .flat(); + const hosts = ['localhost', 'hslocal.net', ...additionalMkcertHosts]; + const { createCertificate } = await import('mkcert-cli'); + const { key, cert } = await silenceConsoleWhile(createCertificate, { + keyFilePath: `${os.tmpdir()}/hstmp/hsLocalSshKey.pem`, + certFilePath: `${os.tmpdir()}/hstmp/hsLocalSshCert.pem` + }, hosts); + unlinkSync(`${os.tmpdir()}/hstmp/hsLocalSshKey.pem`); + unlinkSync(`${os.tmpdir()}/hstmp/hsLocalSshCert.pem`); + return { key, cert }; +} + +const requireHTTPS = (message, response) => { + const newLocation = `https://${message.headers.host}${message.url}`; + + response + .writeHead(302, { + Location: newLocation, + }) + .end(); +} + +// Running http and https servers on the same port pulled from https://stackoverflow.com/a/42019773 +const createHttpsRedirectingServer = async (handler, domainsToProxy) => { + + const { key, cert } = await createCert(domainsToProxy) + + const innerHTTPServer = http.createServer(requireHTTPS); + const innerHTTPSServer = https.createServer({ key, cert }, handler); + + const server = net.createServer(socket => { + socket.once('data', buffer => { + // Pause the socket + socket.pause(); + + // Determine if this is an HTTP(s) request + const byte = buffer[0]; + + let protocol; + if (byte === 22) { + protocol = 'https'; + } else if (32 < byte && byte < 127) { + protocol = 'http'; + } else { + throw new Error( + 'Unknown issue with incoming data, unknown if http or https' + ); + } + + const proxy = protocol === 'http' ? innerHTTPServer : innerHTTPSServer; + if (proxy) { + // Push the buffer back onto the front of the data stream + socket.unshift(buffer); + + // Emit the socket to the HTTP(s) server + proxy.emit('connection', socket); + } + + // As of NodeJS 10.x the socket must be + // resumed asynchronously or the socket + // connection hangs, potentially crashing + // the process. Prior to NodeJS 10.x + // the socket may be resumed synchronously. + process.nextTick(() => socket.resume()); + }); + }); + + return { server, innerHTTPServer, innerHTTPSServer }; +} + +module.exports = { + createHttpsRedirectingServer +} diff --git a/lib/preview/previewUtils.js b/lib/preview/previewUtils.js new file mode 100644 index 0000000..a862236 --- /dev/null +++ b/lib/preview/previewUtils.js @@ -0,0 +1,185 @@ +const { fetchDomains } = require('../../api/domains'); +const { getAccountId, isTrackingAllowed, getAccountConfig } = require('../config'); +const { platform, release } = require('os'); +const { trackUsage } = require('../../api/fileMapper'); +const { enabledFeaturesForPersonalAccessKey } = require('../../personalAccessKey'); +const { stringify } = require('querystring'); +const { logger } = require('./../../logger'); + +const VALID_PROXY_DOMAIN_SUFFIXES = ['localhost', 'hslocal.net']; + +const HS_PREVIEW_GATE = "cms:localHublPreviews"; + +const getPortalDomains = async (accountId) => { + try { + const result = await fetchDomains(accountId); + return result; + } catch (error) { + return []; + } +} + +const getPreviewUrl = (sessionInfo, queryParams) => { + const { accountId, env, hublet } = sessionInfo; + + return `http://${accountId}.hubspotpreview${ + env === 'qa' ? 'qa' : '' + }-${hublet}.com/_hcms/preview/template/multi?${stringify(queryParams)}`; +} + +const insertAtEndOfBody = (html, script) => { + const insertAt = (baseStr, index, insertStr) => { + return `${baseStr.slice(0, index)}${insertStr}${baseStr.slice(index)}`; + } + const endOfBodyIndex = html.lastIndexOf(""); + return insertAt(html, endOfBodyIndex, script); +} + +const addRefreshScript = (html) => { + const refreshScript = ` + + `; + return insertAtEndOfBody(html, refreshScript); +} + +const getSubDomainFromValidLocalDomain = hostname => { + for (const validProxyDomainSuffix of VALID_PROXY_DOMAIN_SUFFIXES) { + if (hostname.endsWith(`.${validProxyDomainSuffix}`)) { + return hostname.slice(0, -1 * validProxyDomainSuffix.length - 1); + } + } +}; + +const internalRoutes = { + HCMS: '/_hcms/', + HS_FS: '/hs-fs/', + HUB_FS: '/hubfs/' +} + +const isInternalCMSRoute = (req) => + Object.values(internalRoutes).some((route => req.path.startsWith(route))); + +const silenceConsoleWhile = async (act, ...args) => { + const tmpConsole = console; + console = { log: () => {} } // ! + const result = await act(...args); + console = tmpConsole; + return result; +} + +const memoize = (func, cacheBustCallback) => { + const cache = {}; + return async (...args) => { + const stringArgs = args.toString(); + const storedResult = cache[stringArgs]; + if (storedResult && (cacheBustCallback ? !cacheBustCallback() : true)) { + //console.log(`Cache hit ${func}`) + return storedResult; + } + //console.log(`Cache miss ${func}`) + const res = await func(...args); + cache[stringArgs] = res; + return res; + } +} + +const hidePreviewInDest = (previewDest) => previewDest.split('/').slice(2).join('/'); + +const trackPreviewEvent = async (action) => { + if (!isTrackingAllowed()) { + return; + } + const accountId = getAccountId(); + + trackUsage( + 'cli-interaction', + 'INTERACTION', + { + applicationName: 'hubspot.preview', + os: `${platform()} ${release()}`, + authType: getAuthType(accountId), + action, + }, + accountId + ).catch( + (err) => { + logger.debug(`trackUsage failed: ${JSON.stringify(err, null, 2)}`); + } + ); +} + +const isUngatedForPreview = async (sessionInfo) => { + const { accountId } = sessionInfo; + + const enabledFeatures = await enabledFeaturesForPersonalAccessKey(accountId); + + return (Object.keys(enabledFeatures).includes(HS_PREVIEW_GATE) + && enabledFeatures[HS_PREVIEW_GATE] === true) +} + +const buildHTMLResponse = (content) => { + return ` + + + + + ${content} + + `; + } + +const getAuthType = accountId => { + let authType = 'unknown'; + + if (accountId) { + const accountConfig = getAccountConfig(accountId); + if (accountConfig && accountConfig.authType) { + authType = accountConfig.authType; + } + } + + return authType; +}; + +module.exports = { + isInternalCMSRoute, + VALID_PROXY_DOMAIN_SUFFIXES, + getSubDomainFromValidLocalDomain, + getPortalDomains, + getPreviewUrl, + addRefreshScript, + silenceConsoleWhile, + memoize, + hidePreviewInDest, + trackPreviewEvent, + isUngatedForPreview, + buildHTMLResponse, +} diff --git a/lib/preview/proxyPage.js b/lib/preview/proxyPage.js new file mode 100644 index 0000000..d2c6c28 --- /dev/null +++ b/lib/preview/proxyPage.js @@ -0,0 +1,27 @@ +const { addRefreshScript } = require('./previewUtils'); +const { fetchPreviewRender } = require('../../api/preview'); + +const proxyPage = async ( + res, + urlToProxy, + sessionInfo +) => { + try { + const pageHtml = await fetchPreviewRender(urlToProxy, sessionInfo); + + const embeddedHtml = addRefreshScript(pageHtml); + res.status(200).set({ 'Content-Type': 'text/html' }).end(embeddedHtml); + } catch (error) { + const { accountId } = sessionInfo; + res + .status(500) + .end( + `Failed proxy render of page ${urlToProxy} hub id = ${accountId}\n\n${error.message}` + ); + return; + } +} + +module.exports = { + proxyPage +} diff --git a/lib/preview/routes/index.js b/lib/preview/routes/index.js new file mode 100644 index 0000000..ade92c0 --- /dev/null +++ b/lib/preview/routes/index.js @@ -0,0 +1,133 @@ +const { + hidePreviewInDest, + trackPreviewEvent, +} = require('../previewUtils'); +const { + fetchPreviewTemplates, + fetchPreviewModules +} = require('../../../api/designManager'); +const { parse: pathParse } = require('path'); +const { logger } = require('./../../../logger'); +const { isCodedFile } = require('./../../../templates'); + +const buildIndexRouteHandler = (sessionInfo) => { + return async (req, res) => { + trackPreviewEvent('view-index-route'); + + const responseHTML = await buildIndexHtml(sessionInfo); + res.status(200).set({ 'Content-Type': 'text/html' }).end(responseHTML); + } +} + +const buildIndexHtml = async (sessionInfo) => { + const { domains, dest, PORT } = sessionInfo; + const fakeDest = hidePreviewInDest(dest); + const modulesHtml = await getModulesForDisplayToUser(sessionInfo); + const templatesHtml = await getTemplatesForDisplayToUser(sessionInfo); + return ` + + + + + +
+

Domains

+ ${ domains.length ? + getSiteList(domains, PORT) : + "

No domains found. You either don't have any domains set up in your portal or your personal access key is missing a scope 'cms.domains.read' required for this feature.

" + } +
+
+
+

Modules

+ ${ modulesHtml ? modulesHtml : `

No modules found in ${fakeDest}

` } +
+
+

Templates

+ ${ templatesHtml ? templatesHtml : `

No templates found in ${fakeDest}

` } +
+
+ + + `; +} + +const getSiteList = (domains, PORT) => { + return listify( + domains, + domainObj => `http://${domainObj.domain}.hslocal.net:${PORT}`, + domainObj => domainObj.domain + ); +} + +const listify = (objects, hrefBuilder, labelBuilder) => { + return ''; +} + +const getModulesForDisplayToUser = async (sessionInfo) => { + const { accountId, sessionToken } = sessionInfo; + try { + const res = await fetchPreviewModules(accountId, sessionToken); + const modulePaths = res + .objects + .map(moduleObj => moduleObj.path); + const filesGroupedByFolder = groupByFolder(modulePaths) + const htmlToRender = renderFilesGroupedByFolder(filesGroupedByFolder, 'module') + return htmlToRender; + } catch (err) { + logger.error(`Failed to fetch modules for index page: ${err}`); + return undefined; + } +} + +const getTemplatesForDisplayToUser = async (sessionInfo) => { + const { accountId, sessionToken } = sessionInfo; + try { + const res = await fetchPreviewTemplates(accountId, sessionToken); + const templatePaths = res + .objects + .filter(templateObj => isCodedFile(templateObj.filename)) + .map(templateObj => templateObj.path); + const filesGroupedByFolder = groupByFolder(templatePaths) + const htmlToRender = renderFilesGroupedByFolder(filesGroupedByFolder, 'template') + return htmlToRender; + } catch (err) { + logger.error(`Failed to fetch templates for index page: ${err}`); + } +} + +const groupByFolder = (paths) => { + return paths.reduce((acc, cur) => { + const { dir, base } = pathParse(cur); + if (Object.keys(acc).includes(dir)) { + acc[dir].push(base); + } else { + acc[dir] = [base]; + } + return acc; + }, {}) +} + +const renderFilesGroupedByFolder = (filesGroupedByFolder, endpoint) => { + const folders = Object.keys(filesGroupedByFolder); + return folders.reduce((outer_acc, folder) => { + const fakeDest = hidePreviewInDest(folder); + const folderDisplay = '' + + fakeDest + + '' + + listify( + filesGroupedByFolder[folder], + x => `/${endpoint}/${fakeDest}/${x}`, + x => x + ); + outer_acc += folderDisplay; + return outer_acc; + }, ''); +} + + +module.exports = { + buildIndexRouteHandler +} diff --git a/lib/preview/routes/meta.js b/lib/preview/routes/meta.js new file mode 100644 index 0000000..75a026b --- /dev/null +++ b/lib/preview/routes/meta.js @@ -0,0 +1,19 @@ +const hsServerState = { + REMOTE_FS_IS_DIRTY: false +} + +const buildMetaRouteHandler = (sessionInfo) => { + return async (req, res) => { + res.json(hsServerState); + hsServerState["REMOTE_FS_IS_DIRTY"] = false; + } +} + +const markRemoteFsDirty = () => { + hsServerState["REMOTE_FS_IS_DIRTY"] = true; +} + +module.exports = { + buildMetaRouteHandler, + markRemoteFsDirty, +} diff --git a/lib/preview/routes/module.js b/lib/preview/routes/module.js new file mode 100644 index 0000000..292774e --- /dev/null +++ b/lib/preview/routes/module.js @@ -0,0 +1,70 @@ +const http = require('../../../http') +const { fetchModulesByPath } = require('../../../api/designManager'); +const { + addRefreshScript, + getPreviewUrl, + trackPreviewEvent, + buildHTMLResponse +} = require('../previewUtils'); +const { logger } = require('./../../../logger'); + +const buildModuleRouteHandler = (sessionInfo) => { + const { accountId, sessionToken } = sessionInfo; + + return async (req, res) => { + trackPreviewEvent('view-module-route'); + + const { modulePath } = req.params; + + if (!modulePath) { + res.status(200).set({ 'Content-Type': 'text/html' }).end(buildModuleIndex()); + return; + } + + const calculatedPath = `@preview/${sessionToken}/${modulePath}.module`; + let customWidgetInfo; + try { + customWidgetInfo = await fetchModulesByPath(accountId, calculatedPath); + } catch (err) { + logger.error(`Failed to fetch module preview for ${calculatedPath}`) + } + if (!customWidgetInfo || !('moduleId' in customWidgetInfo && 'previewKey' in customWidgetInfo)) { + res.status(200).set({ 'Content-Type': 'text/html' }).end(buildErrorIndex()); + return; + } + const params = { + module_id: customWidgetInfo.moduleId, + hs_preview_key: customWidgetInfo.previewKey, + updated: Date.now(), + ...req.query + } + const previewUrl = new URL(getPreviewUrl(sessionInfo, params)) ; + const result = await http.get(accountId, { + baseUrl: previewUrl.origin, + uri: previewUrl.pathname, + query: params, + json: false, + useQuerystring: true, + }); + const html = addRefreshScript(result) + res.status(200).set({ 'Content-Type': 'text/html' }).end(html); + } +} + +const buildErrorIndex = () => buildHTMLResponse(` +
+

Error

+

Failed to fetch module data.

+
+`); + +const buildModuleIndex = () => buildHTMLResponse(` +
+

Module

+

Please provide a module path in the request

+
+`); + +module.exports = { + buildModuleRouteHandler +} diff --git a/lib/preview/routes/proxyPageResourceRedirect.js b/lib/preview/routes/proxyPageResourceRedirect.js new file mode 100644 index 0000000..143651d --- /dev/null +++ b/lib/preview/routes/proxyPageResourceRedirect.js @@ -0,0 +1,59 @@ +const request = require('request'); +const { + getSubDomainFromValidLocalDomain, + VALID_PROXY_DOMAIN_SUFFIXES, + isInternalCMSRoute +} = require('../previewUtils'); + +const proxyPageResourceRedirect = (req, res, next) => { + const isAHtmlRequest = req.accepts().includes('text/html'); + + if ( + (isAHtmlRequest && !isInternalCMSRoute(req)) || + VALID_PROXY_DOMAIN_SUFFIXES.includes(req.hostname) + ) { + next(); + return; + } + + const domainToProxy = getSubDomainFromValidLocalDomain(req.hostname); + const pathToProxy = req.originalUrl; + const urlToProxy = `https://${domainToProxy}${pathToProxy}`; + + if (req.method === 'GET') { + request(urlToProxy).pipe(res); + } else { + const contentType = req.headers['content-type']; + const isSendingJSON = + contentType && /\bjson\b/.test(req.headers['content-type'].toString()); + + const headerEntries = Object.entries(req.headers).filter(([header]) => + // Need to reset these headers for https (and recompression/encoding?) to work properly + !['host', 'origin', 'connection', 'content-length'].includes( + header.toLowerCase() + ) && + !header.startsWith('sec-') + ); + + let body; + + if (req.body) { + if (isSendingJSON) { + body = JSON.stringify(req.body); + } else { + body = req.body.toString(); + } + } + + request({ + url: urlToProxy, + method: req.method, + headers: Object.fromEntries(headerEntries), + body, + }).pipe(res); + } +} + +module.exports = { + proxyPageResourceRedirect +} diff --git a/lib/preview/routes/proxyPageRouteHandler.js b/lib/preview/routes/proxyPageRouteHandler.js new file mode 100644 index 0000000..7873645 --- /dev/null +++ b/lib/preview/routes/proxyPageRouteHandler.js @@ -0,0 +1,43 @@ +const { proxyPage } = require('../proxyPage.js'); +const { + getSubDomainFromValidLocalDomain, + VALID_PROXY_DOMAIN_SUFFIXES, + trackPreviewEvent +} = require('../previewUtils.js'); + +const buildProxyPageRouteHandler = (sessionInfo) => { + return async (request, response, next) => { + if (!request.accepts().includes('text/html')) { + return response.sendStatus(404); + } + + const domainToProxy = getSubDomainFromValidLocalDomain(request.hostname); + + if (!domainToProxy) { + if (VALID_PROXY_DOMAIN_SUFFIXES.includes(request.hostname)) { + next(); + return; + } else { + const message = `Can't make a proxy request, the domain '${request.hostname}' is not a valid proxy-able domain`; + console.warn(message); + return response.status(400).send(message); + } + } + trackPreviewEvent('proxy-page-route') + const urlToProxy = `http://${domainToProxy}${request.originalUrl}`; + + try { + await proxyPage( + response, + urlToProxy, + sessionInfo, + ); + } catch (e) { + next(e); + } + } +} + +module.exports = { + buildProxyPageRouteHandler, +} diff --git a/lib/preview/routes/proxyPathPageResourceRedirect.js b/lib/preview/routes/proxyPathPageResourceRedirect.js new file mode 100644 index 0000000..b1a6d13 --- /dev/null +++ b/lib/preview/routes/proxyPathPageResourceRedirect.js @@ -0,0 +1,41 @@ +const { isInternalCMSRoute } = require('../previewUtils'); +const request = require('request'); + +const proxyPathPageResourceRedirect = (req, res, next) => { + const isAHtmlRequest = req.accepts().includes('text/html'); + + // proxy when referer's path is /proxy and has page query param + let proxyPageUrl; + let refererUrl; + try { + refererUrl = new URL(req.headers.referer); + proxyPageUrl = new URL( + refererUrl.searchParams.get('page') + ); + } catch (e) { + next(); + return; + } + if (!refererUrl.pathname.startsWith('/proxy')) { + next(); + return; + } + + // handle anchor links unless internal CMS route + if (isAHtmlRequest && !isInternalCMSRoute(req)) { + const encodedPageUrl = encodeURIComponent( + `${proxyPageUrl.origin}${req.url}` + ); + res.redirect(`/proxy?page=${encodedPageUrl}`); + return; + } + + const urlToProxy = `https://${proxyPageUrl.host}${req.url}`; + request(urlToProxy) + .then(response => response.body.pipe(res)) + .catch(err => console.error(err)); +} + +module.exports = { + proxyPathPageResourceRedirect, +} diff --git a/lib/preview/routes/proxyPathPageRouteHandler.js.js b/lib/preview/routes/proxyPathPageRouteHandler.js.js new file mode 100644 index 0000000..efad899 --- /dev/null +++ b/lib/preview/routes/proxyPathPageRouteHandler.js.js @@ -0,0 +1,63 @@ +const { proxyPage } = require('../proxyPage'); +const { trackPreviewEvent, buildHTMLResponse } = require('../previewUtils'); +const { logger } = require('./../../../logger'); + +const buildProxyRouteHandler = (sessionInfo) => { + + return async (req, res) => { + if (!req.query.page) { + res.status(200).set({ 'Content-Type': 'text/html' }).end(buildProxyIndex()); + return; + } + + // parse proxy page URL from query param + let proxyPageUrl; + try { + proxyPageUrl = new URL(req.query.page); + } catch (e) { + const message = + 'Please provide a valid page query parameter, e.g., http://localhost:3000/proxy?page=https://yourdomain.com/path'; + logger.warn(message); + return res.status(400).send(message); + } + // validate proxy page URL query params + if (proxyPageUrl.searchParams.has('hs_preview')) { + const message = `Can't make a proxy request, you cannot proxy URLs that include internal query params like hs_preview`; + logger.warn(message); + return res.status(400).send(message); + } + trackPreviewEvent('proxy-path-route'); + try { + await proxyPage( + res, + proxyPageUrl.href, + sessionInfo + ); + } catch (e) { + const message = `Failed to fetch proxy page for ${proxyPageUrl.href}`; + return res.status(400).send(message); + } + } +} + +const buildProxyIndex = () => buildHTMLResponse(` +
+

Local Proxy

+
+ + + +
+
+`); + +module.exports = { + buildProxyRouteHandler +} diff --git a/lib/preview/routes/template.js b/lib/preview/routes/template.js new file mode 100644 index 0000000..2241ffa --- /dev/null +++ b/lib/preview/routes/template.js @@ -0,0 +1,62 @@ +const http = require('../../../http'); +const { fetchTemplatesByPath } = require('../../../api/designManager'); +const { getPreviewUrl, addRefreshScript, buildHTMLResponse } = require('../previewUtils'); +const { logger } = require('./../../../logger'); + +const buildTemplateRouteHandler = (sessionInfo) => { + const { accountId, sessionToken } = sessionInfo; + + return async (req, res) => { + const { templatePath } = req.params; + + if (!templatePath) { + res.status(200).set({ 'Content-Type': 'text/html' }).end(buildTemplateIndex()); + return; + } + + const calculatedPath = `@preview/${sessionToken}/${templatePath}`; + let templateInfo; + try { + templateInfo = await fetchTemplatesByPath(accountId, calculatedPath); + } catch (err) { + logger.error(`Failed to fetch template info for ${calculatedPath}: ${err}`); + } + if (!templateInfo || !('previewKey' in templateInfo)) { + res.status(502).set({ 'Content-Type': 'text/html' }).end(buildErrorIndex()); + return; + } + const params = { + template_file_path: calculatedPath, + hs_preview_key: templateInfo.previewKey, + ...req.query + } + const previewUrl = new URL(getPreviewUrl(sessionInfo, params)); + const result = await http.get(accountId, { + baseUrl: previewUrl.origin, + uri: previewUrl.pathname, + query: params, + json: false, + useQuerystring: true, + }); + const html = addRefreshScript(result) + res.status(200).set({ 'Content-Type': 'text/html' }).end(html); + } +} + +const buildErrorIndex = () => buildHTMLResponse(` +
+

Error

+

Failed to fetch template data.

+
+`) + +const buildTemplateIndex = () => buildHTMLResponse(` +
+

Template

+

Please provide a template path in the request

+
+`) + +module.exports = { + buildTemplateRouteHandler +} diff --git a/lib/preview/sprocketMenuServer.js b/lib/preview/sprocketMenuServer.js new file mode 100644 index 0000000..8b4fad3 --- /dev/null +++ b/lib/preview/sprocketMenuServer.js @@ -0,0 +1,83 @@ +const net = require('net'); +const express = require('express'); +const { Router } = require('express'); +const cors = require("cors"); +const { logger } = require('../../logger'); + +const SPROCKET_MENU_PORT = 1442; + +const startSprocketMenuServer = async (sessionInfo) => { + const portIsTaken = await new Promise((res, rej) => { + const testNetServer = net.createServer(); + + testNetServer.once('error', err => { + if (err['code'] === 'EADDRINUSE') { + logger.error( + `Port ${SPROCKET_MENU_PORT} is in use. HubSpot will be unable to automatically create proxy links in the Sprocket Menu` + ); + res(true); + } else { + rej(err); + } + }); + + testNetServer.once('listening', () => { + testNetServer.close(); + }); + + testNetServer.once('close', () => { + res(false); + }); + + testNetServer.listen(SPROCKET_MENU_PORT); + }); + + if (portIsTaken) { + return; + } + + const sprocketMenuServer = express(); + + sprocketMenuServer.listen(SPROCKET_MENU_PORT); + + sprocketMenuServer.use( + '/', + await createSprocketMenuServerRoutes(sessionInfo) + ); +} + +const createSprocketMenuServerRoutes = async (sessionInfo) => { + const sprocketMenuServerRoutes = Router(); + + sprocketMenuServerRoutes.get( + '/check-if-local-dev-server', + cors(), + sprocketMenuServerCheckHandler(sessionInfo), + ); + + return sprocketMenuServerRoutes; +} + +const sprocketMenuServerCheckHandler = (sessionInfo) => async (req, res) => { + const { PORT } = sessionInfo; + const { query } = req; + + if (query) { + const { + hostName, + pathName, + } = query; + + let hasJSBuildingBlocks = false; + let localProxyUrl = `http://${hostName}.hslocal.net:${PORT}${pathName}`; + + res + .status(200) + .set({ 'Content-Type': 'application/json' }) + .json({ hasJSBuildingBlocks, localProxyUrl }); +}; +} + +module.exports = { + startSprocketMenuServer +} diff --git a/lib/uploadFolder.js b/lib/uploadFolder.js index ab5d5a9..2d1f96c 100644 --- a/lib/uploadFolder.js +++ b/lib/uploadFolder.js @@ -107,6 +107,46 @@ async function getFilesByType( return [filePathsByType, fieldsJsObjects]; } +const defaultUploadAttemptCallback = ((file, destPath) => + logger.debug( + 'Attempting to upload file "%s" to "%s"', + file, + destPath + ) +); +const defaultUploadSuccessCallback = (file, destPath) => logger.log( + 'Uploaded file "%s" to "%s"', + file, + destPath +); +const defaultUploadFirstErrorCallback = (file, destPath, error) => { + logger.debug( + 'Uploading file "%s" to "%s" failed so scheduled retry', + file, + destPath + ); + if (error.response && error.response.body) { + logger.debug(error.response.body); + } else { + logger.debug(error.message); + } +}; +const defaultUploadRetryCallback = (file, destPath) => logger.debug( + 'Retrying to upload file "%s" to "%s"', + file, + destPath +); +const defaultUploadFinalErrorCallback = (accountId, file, destPath, error) => { + logger.error('Uploading file "%s" to "%s" failed', file, destPath); + logApiUploadErrorInstance( + error, + new ApiErrorContext({ + accountId, + request: destPath, + payload: file, + }) + ); +}; /** * * @param {number} accountId @@ -122,7 +162,20 @@ async function uploadFolder( commandOptions = {}, filePaths = [] ) { - const { saveOutput, convertFields } = commandOptions; + const { + saveOutput, + convertFields, + onAttemptCallback, + onSuccessCallback, + onFirstErrorCallback, + onRetryCallback, + onFinalErrorCallback + } = commandOptions; + const _onAttemptCallback = onAttemptCallback || defaultUploadAttemptCallback; + const _onSuccessCallback = onSuccessCallback || defaultUploadSuccessCallback; + const _onFirstErrorCallback = onFirstErrorCallback || defaultUploadFirstErrorCallback; + const _onRetryCallback = onRetryCallback || defaultUploadRetryCallback; + const _onFinalErrorCallback = onFinalErrorCallback || defaultUploadFinalErrorCallback; const tmpDir = convertFields ? createTmpDirSync('hubspot-temp-fieldsjs-output-') : null; @@ -162,28 +215,15 @@ async function uploadFolder( ); const destPath = convertToUnixPath(path.join(dest, relativePath)); return async () => { - logger.debug( - 'Attempting to upload file "%s" to "%s"', - originalFilePath, - destPath - ); + _onAttemptCallback(originalFilePath, destPath); try { await upload(accountId, file, destPath, apiOptions); - logger.log('Uploaded file "%s" to "%s"', originalFilePath, destPath); + _onSuccessCallback(originalFilePath, destPath); } catch (error) { if (isFatalError(error)) { throw error; } - logger.debug( - 'Uploading file "%s" to "%s" failed so scheduled retry', - file, - destPath - ); - if (error.response && error.response.body) { - logger.debug(error.response.body); - } else { - logger.debug(error.message); - } + _onFirstErrorCallback(file, destPath, error); failures.push({ file, destPath, @@ -202,28 +242,20 @@ async function uploadFolder( .addAll( failures.map(({ file, destPath }) => { return async () => { - logger.debug('Retrying to upload file "%s" to "%s"', file, destPath); + _onRetryCallback(file, destPath); try { await upload(accountId, file, destPath, apiOptions); - logger.log('Uploaded file "%s" to "%s"', file, destPath); + _onSuccessCallback(file, destPath); return { resultType: FileUploadResultType.SUCCESS, error: null, file, }; } catch (error) { - logger.error('Uploading file "%s" to "%s" failed', file, destPath); if (isFatalError(error)) { throw error; } - logApiUploadErrorInstance( - error, - new ApiErrorContext({ - accountId, - request: destPath, - payload: file, - }) - ); + _onFinalErrorCallback(accountId, file, destPath, error); return { resultType: FileUploadResultType.FAILURE, error, diff --git a/package.json b/package.json index a0b3604..44d346b 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "scripts": { "check-main": "branch=$(git rev-parse --abbrev-ref HEAD) && [ $branch = main ] || (echo 'Error: New release can only be published on main branch' && exit 1)", "lint": "eslint . && prettier --list-different packages/**/*.{js,json}", - "prettier:write": "prettier --write packages/**/*.{js,json}", + "prettier:write": "prettier --write **/*.{js,json}", "pub": "npm publish --tag latest", "push": "git push --atomic origin main v$npm_package_version", "release:major": "yarn check-main && yarn version --major && yarn pub && yarn push", @@ -24,13 +24,16 @@ "chalk": "^2.4.2", "chokidar": "^3.0.1", "content-disposition": "^0.5.3", + "cors": "^2.8.5", "debounce": "^1.2.0", + "express": "^4.18.2", "extract-zip": "^1.6.7", "findup-sync": "^4.0.0", "fs-extra": "^8.1.0", "ignore": "^5.1.4", "jest": "^29.5.0", "js-yaml": "^4.1.0", + "mkcert-cli": "^1.5.0", "moment": "^2.24.0", "p-queue": "^6.0.2", "prettier": "^1.19.1", diff --git a/personalAccessKey.js b/personalAccessKey.js index 07dc2eb..39e0d1f 100644 --- a/personalAccessKey.js +++ b/personalAccessKey.js @@ -49,6 +49,7 @@ async function getAccessToken( accessToken: response.oauthAccessToken, expiresAt: moment(response.expiresAtMillis), scopeGroups: response.scopeGroups, + enabledFeatures: response.enabledFeatures, encodedOauthRefreshToken: response.encodedOauthRefreshToken, }; } @@ -58,11 +59,12 @@ async function refreshAccessToken( personalAccessKey, env = ENVIRONMENTS.PROD ) { - const { accessToken, expiresAt } = await getAccessToken( + const accessTokenResponse = await getAccessToken( personalAccessKey, env, accountId ); + const { accessToken, expiresAt } = accessTokenResponse const config = getAccountConfig(accountId); updateAccountConfig({ @@ -75,15 +77,15 @@ async function refreshAccessToken( }); writeConfig(); - return accessToken; + return accessTokenResponse; } -async function getNewAccessToken(accountId, personalAccessKey, expiresAt, env) { +async function getNewAccessToken(accountId, personalAccessKey, expiresAt, env, fullAPIResponse=false) { const key = getRefreshKey(personalAccessKey, expiresAt); if (refreshRequests.has(key)) { return refreshRequests.get(key); } - let accessToken; + let accessTokenResponse; try { const refreshAccessPromise = refreshAccessToken( accountId, @@ -93,14 +95,17 @@ async function getNewAccessToken(accountId, personalAccessKey, expiresAt, env) { if (key) { refreshRequests.set(key, refreshAccessPromise); } - accessToken = await refreshAccessPromise; + accessTokenResponse = await refreshAccessPromise; } catch (e) { if (key) { refreshRequests.delete(key); } throw e; } - return accessToken; + if (fullAPIResponse) { + return accessTokenResponse; + } + return accessTokenResponse.accessToken; } /** @@ -129,6 +134,20 @@ async function accessTokenForPersonalAccessKey(accountId) { return auth.tokenInfo.accessToken; } +async function enabledFeaturesForPersonalAccessKey(accountId) { + const { auth, personalAccessKey, env } = getAccountConfig(accountId); + const authTokenInfo = auth && auth.tokenInfo; + + const accessTokenResponse = await getNewAccessToken( + accountId, + personalAccessKey, + authTokenInfo && authTokenInfo.expiresAt, + env, + fullAPIResponse=true + ) + return accessTokenResponse.enabledFeatures; +} + /** * @deprecated @@ -194,6 +213,7 @@ const updateConfigWithPersonalAccessKey = async (configData, makeDefault) => { module.exports = { accessTokenForPersonalAccessKey, + enabledFeaturesForPersonalAccessKey, updateConfigWithPersonalAccessKey, getAccessToken, };