Merge pull request #78 from parcelvoy/feat/add-analytics-providers

Add analytics providers
This commit is contained in:
Chris Hills 2023-03-15 19:21:21 -05:00 committed by GitHub
commit cd309696f6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
67 changed files with 612 additions and 111 deletions

View file

@ -8,7 +8,6 @@
"name": "platform",
"version": "0.1.0",
"dependencies": {
"@apideck/better-ajv-errors": "^0.3.6",
"@aws-sdk/client-s3": "^3.171.0",
"@aws-sdk/client-ses": "^3.121.0",
"@aws-sdk/client-sns": "^3.121.0",
@ -19,7 +18,9 @@
"@ladjs/country-language": "^1.0.3",
"@node-saml/node-saml": "github:node-saml/node-saml",
"@rxfork/sqs-consumer": "^6.0.0",
"@segment/analytics-node": "^1.0.0-beta.23",
"ajv": "^8.11.0",
"ajv-errors": "^3.0.0",
"ajv-formats": "^2.1.1",
"busboy": "^1.6.0",
"csv-parse": "^5.3.3",
@ -81,22 +82,6 @@
"node": ">=6.0.0"
}
},
"node_modules/@apideck/better-ajv-errors": {
"version": "0.3.6",
"resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz",
"integrity": "sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA==",
"dependencies": {
"json-schema": "^0.4.0",
"jsonpointer": "^5.0.0",
"leven": "^3.1.0"
},
"engines": {
"node": ">=10"
},
"peerDependencies": {
"ajv": ">=8"
}
},
"node_modules/@aws-crypto/crc32": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz",
@ -2744,6 +2729,25 @@
"node": ">= 14"
}
},
"node_modules/@lukeed/csprng": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.0.1.tgz",
"integrity": "sha512-uSvJdwQU5nK+Vdf6zxcWAY2A8r7uqe+gePwLWzJ+fsQehq18pc0I2hJKwypZ2aLM90+Er9u1xn4iLJPZ+xlL4g==",
"engines": {
"node": ">=8"
}
},
"node_modules/@lukeed/uuid": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@lukeed/uuid/-/uuid-2.0.0.tgz",
"integrity": "sha512-dUz8OmYvlY5A9wXaroHIMSPASpSYRLCqbPvxGSyHguhtTQIy24lC+EGxQlwv71AhRCO55WOtgwhzQLpw27JaJQ==",
"dependencies": {
"@lukeed/csprng": "^1.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@node-saml/node-saml": {
"version": "4.0.3",
"resolved": "git+ssh://git@github.com/node-saml/node-saml.git#f86252bcbaa4435159d121d2e342ec89f94ad183",
@ -2841,6 +2845,62 @@
"@aws-sdk/client-sqs": "^3.5.0"
}
},
"node_modules/@segment/analytics-core": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@segment/analytics-core/-/analytics-core-1.2.2.tgz",
"integrity": "sha512-zVWSDcyh7Rp32xL5v2fuEk2yZxxy+JA93vF1L3EF9XAYLSra/uEHJEswOWieXSdDHVRHes7APORp136usFE/tw==",
"dependencies": {
"@lukeed/uuid": "^2.0.0",
"dset": "^3.1.2",
"tslib": "^2.4.1"
}
},
"node_modules/@segment/analytics-node": {
"version": "1.0.0-beta.23",
"resolved": "https://registry.npmjs.org/@segment/analytics-node/-/analytics-node-1.0.0-beta.23.tgz",
"integrity": "sha512-S1P6opKo1WfBxzX2MeBqFmZ9ViMUmP3RpXNFqeCnkM2c7hqY3FEZJNV7CrOdH3ej5W1S8cUE2/U6dJgzt6jv3g==",
"dependencies": {
"@segment/analytics-core": "1.2.2",
"buffer": "^6.0.3",
"node-fetch": "^2.6.7",
"tslib": "^2.4.1",
"uuid": "^9.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@segment/analytics-node/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/@segment/analytics-node/node_modules/uuid": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
"integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/@sinclair/typebox": {
"version": "0.24.51",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.51.tgz",
@ -3599,6 +3659,14 @@
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/ajv-errors": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-3.0.0.tgz",
"integrity": "sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==",
"peerDependencies": {
"ajv": "^8.0.1"
}
},
"node_modules/ajv-formats": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
@ -5335,6 +5403,14 @@
"node": ">=12"
}
},
"node_modules/dset": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/dset/-/dset-3.1.2.tgz",
"integrity": "sha512-g/M9sqy3oHe477Ar4voQxWtaPIFw1jTdKZuomOjhCcBx9nHUNn0pu6NopuFFrTh/TRZIKEj+76vLWFu9BNKk+Q==",
"engines": {
"node": ">=4"
}
},
"node_modules/ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@ -8877,14 +8953,6 @@
"underscore": "1.12.1"
}
},
"node_modules/jsonpointer": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz",
"integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/jsonwebtoken": {
"version": "8.5.1",
"resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz",
@ -9249,6 +9317,7 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz",
"integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==",
"dev": true,
"engines": {
"node": ">=6"
}
@ -9762,6 +9831,25 @@
"node": ">= 0.6.0"
}
},
"node_modules/node-fetch": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz",
"integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/node-forge": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz",
@ -12218,6 +12306,11 @@
"node": ">=0.8"
}
},
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/ts-jest": {
"version": "28.0.8",
"resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-28.0.8.tgz",
@ -12752,6 +12845,20 @@
"safe-buffer": "^5.0.1"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

View file

@ -12,6 +12,7 @@
"@ladjs/country-language": "^1.0.3",
"@node-saml/node-saml": "github:node-saml/node-saml",
"@rxfork/sqs-consumer": "^6.0.0",
"@segment/analytics-node": "^1.0.0-beta.23",
"ajv": "^8.11.0",
"ajv-errors": "^3.0.0",
"ajv-formats": "^2.1.1",

View file

@ -33,7 +33,12 @@ export default class App {
const auth = loadAuth(env.auth)
// Setup app
App.$main = new App(env, database, queue, auth, storage)
App.$main = new App(env,
database,
queue,
auth,
storage,
)
return App.$main
}

View file

@ -1,4 +1,4 @@
import Provider from '../channels/Provider'
import Provider from '../providers/Provider'
import { ChannelType } from '../config/channels'
import Model, { ModelParams } from '../core/Model'
import List from '../lists/List'

View file

@ -1,21 +1,22 @@
import EmailJob from '../channels/email/EmailJob'
import TextJob from '../channels/text/TextJob'
import WebhookJob from '../channels/webhook/WebhookJob'
import { UserEvent } from '../users/UserEvent'
import PushJob from '../providers/push/PushJob'
import WebhookJob from '../providers/webhook/WebhookJob'
import TextJob from '../providers/text/TextJob'
import EmailJob from '../providers/email/EmailJob'
import { User } from '../users/User'
import { UserEvent } from '../users/UserEvent'
import Campaign, { CampaignDelivery, CampaignParams, CampaignSend, CampaignSendParams, CampaignSendState, CampaignState, SentCampaign } from './Campaign'
import { UserList } from '../lists/List'
import Subscription from '../subscriptions/Subscription'
import { RequestError } from '../core/errors'
import App from '../app'
import PushJob from '../channels/push/PushJob'
import { SearchParams } from '../core/searchParams'
import { getList, listUserCount } from '../lists/ListService'
import { allTemplates, duplicateTemplate, validateTemplates } from '../render/TemplateService'
import { utcToZonedTime } from 'date-fns-tz'
import { getSubscription } from '../subscriptions/SubscriptionService'
import { getProvider } from '../channels/ProviderRepository'
import { pick } from '../utilities'
import { getProvider } from '../providers/ProviderRepository'
import { createTagSubquery, getTags, setTags } from '../tags/TagService'
import { getProject } from '../projects/ProjectService'
@ -277,10 +278,10 @@ export const duplicateCampaign = async (campaign: Campaign) => {
}
export const campaignProgress = async (campaign: Campaign): Promise<CampaignDelivery> => {
const progress = await CampaignSend.first(
qb => qb.where('campaign_id', campaign.id)
.select(CampaignSend.raw("SUM(IF(state = 'sent', 1, 0)) AS sent, COUNT(*) AS total")),
) as any
const progress = await CampaignSend.query()
.where('campaign_id', campaign.id)
.select(CampaignSend.raw("SUM(IF(state = 'sent', 1, 0)) AS sent, COUNT(*) AS total"))
.first()
return {
sent: parseInt(progress.sent),
total: parseInt(progress.total),

View file

@ -1,5 +1,5 @@
import App from '../../app'
import EmailJob from '../../channels/email/EmailJob'
import EmailJob from '../../providers/email/EmailJob'
import { RequestError } from '../../core/errors'
import { addUserToList, createList } from '../../lists/ListService'
import { createProject } from '../../projects/ProjectService'
@ -8,7 +8,7 @@ import { User } from '../../users/User'
import { uuid } from '../../utilities'
import Campaign, { CampaignSend, SentCampaign } from '../Campaign'
import { allCampaigns, createCampaign, getCampaign, sendCampaign, sendList } from '../CampaignService'
import { createProvider } from '../../channels/ProviderRepository'
import { createProvider } from '../../providers/ProviderRepository'
import { Admin } from '../../auth/Admin'
afterEach(() => {
@ -39,6 +39,7 @@ describe('CampaignService', () => {
group: 'email',
data: {},
name: uuid(),
is_default: false,
})
return {
project_id: project.id,

View file

@ -27,3 +27,40 @@ export type ClientPostEvent = {
} & ClientIdentity
export type ClientPostEventsRequest = ClientPostEvent[]
export interface SegmentContext {
app?: {
build: string
name: string
namespace: string
version: string
}
ip?: number
os: {
name: string
version: string
}
timezone: string
}
export type SegmentPostEvent = {
event: string
anonymousId: string
userId: string
context: Record<string, any> & SegmentContext
properties: Record<string, any>
traits?: Record<string, any>
type: 'track' | 'alias' | 'identify'
timestamp: string
} & (
{
type: 'track',
properties: Record<string, any>
}
| {
type: 'identify' | 'alias'
traits: Record<string, any>
}
)
export type SegmentPostEventsRequest = SegmentPostEvent[]

View file

@ -1,10 +1,10 @@
import { createEvent } from '../users/UserEventRepository'
import { getUserFromClientId } from '../users/UserRepository'
import { updateUsersLists } from '../lists/ListService'
import { ClientIdentity, ClientPostEvent } from './Client'
import { Job } from '../queue'
import { updateUsersJourneys } from '../journey/JourneyService'
import { logger } from '../config/logger'
import { createAndFetchEvent } from '../users/UserEventRepository'
interface EventPostTrigger {
project_id: number
@ -27,9 +27,7 @@ export default class EventPostJob extends Job {
}
// Create event for given user
const dbEvent = await createEvent({
project_id,
user_id: user.id,
const dbEvent = await createAndFetchEvent(user, {
name: event.name,
data: event.data || {},
})

View file

@ -0,0 +1,98 @@
import Router from '@koa/router'
import App from '../app'
import EventPostJob from './EventPostJob'
import { JSONSchemaType, validate } from '../core/validate'
import { SegmentPostEventsRequest } from './Client'
import { aliasUser } from '../users/UserRepository'
import { ProjectState } from '../auth/AuthMiddleware'
import { projectMiddleware } from '../projects/ProjectController'
import UserPatchJob from '../users/UserPatchJob'
const router = new Router<ProjectState>()
router.use(projectMiddleware)
const segmentEventsRequest: JSONSchemaType<SegmentPostEventsRequest> = {
$id: 'segmentPostEvents',
type: 'array',
items: {
type: 'object',
required: ['event', 'type'],
properties: {
event: { type: 'string' },
type: { type: 'string' },
anonymousId: {
type: 'string',
nullable: true,
},
externalId: {
type: 'string',
nullable: true,
},
properties: {
type: 'object',
nullable: true,
additionalProperties: true,
},
traits: {
type: 'object',
nullable: true,
additionalProperties: true,
},
context: {
type: 'object',
nullable: true,
additionalProperties: true,
},
timestamp: { type: 'string' },
},
anyOf: [
{
required: ['anonymousId'],
},
{
required: ['externalId'],
},
],
},
minItems: 1,
maxItems: 200,
} as any
router.post('/events', async ctx => {
const events = validate(segmentEventsRequest, ctx.request.body)
for (const event of events) {
const identity = {
anonymous_id: event.anonymousId,
external_id: event.userId,
}
if (event.type === 'alias') {
await aliasUser(ctx.state.project.id, identity)
} else if (event.type === 'identify') {
await App.main.queue.enqueue(UserPatchJob.from({
project_id: ctx.state.project.id,
user: {
...identity,
email: event.traits.email,
phone: event.traits.phone,
data: event.traits,
},
}))
} else if (event.type === 'track') {
await App.main.queue.enqueue(EventPostJob.from({
project_id: ctx.state.project.id,
event: {
...identity,
name: event.event,
data: { ...event.properties, ...event.context },
},
}))
}
}
ctx.status = 204
ctx.body = ''
})
export default router

View file

@ -1,7 +1,7 @@
import EmailChannel from '../channels/email/EmailChannel'
import TextChannel from '../channels/text/TextChannel'
import WebhookChannel from '../channels/webhook/WebhookChannel'
import PushChannel from '../channels/push/PushChannel'
import EmailChannel from '../providers/email/EmailChannel'
import TextChannel from '../providers/text/TextChannel'
import WebhookChannel from '../providers/webhook/WebhookChannel'
import PushChannel from '../providers/push/PushChannel'
export type Channel = EmailChannel | TextChannel | PushChannel | WebhookChannel
export type ChannelType = 'email' | 'push' | 'text' | 'webhook'

View file

@ -1,13 +1,14 @@
import Router from '@koa/router'
import ProjectController, { ProjectSubrouter, projectMiddleware } from '../projects/ProjectController'
import ClientController from '../client/ClientController'
import SegmentController from '../client/SegmentController'
import CampaignController from '../campaigns/CampaignController'
import ListController from '../lists/ListController'
import SubscriptionController, { publicRouter as PublicSubscriptionController } from '../subscriptions/SubscriptionController'
import JourneyController from '../journey/JourneyController'
import ImageController from '../storage/ImageController'
import AuthController from '../auth/AuthController'
import ProviderController from '../channels/ProviderController'
import ProviderController from '../providers/ProviderController'
import LinkController from '../render/LinkController'
import TemplateController from '../render/TemplateController'
import UserController from '../users/UserController'
@ -90,6 +91,7 @@ export const clientRouter = () => {
const router = new Router({ prefix: '/client' })
router.use(authMiddleware)
register(router, ClientController)
register(router, SegmentController)
// Secret client routes
router.use(scopeMiddleware('secret'))

View file

@ -1,6 +1,6 @@
import pino from 'pino'
import pretty from 'pino-pretty'
import Provider from '../channels/Provider'
import Provider from '../providers/Provider'
import { DriverConfig } from './env'
export type LoggerProviderName = 'logger'

View file

@ -1,10 +1,10 @@
import Queue from '../queue'
import EmailJob from '../channels/email/EmailJob'
import EmailJob from '../providers/email/EmailJob'
import EventPostJob from '../client/EventPostJob'
import TextJob from '../channels/text/TextJob'
import TextJob from '../providers/text/TextJob'
import UserDeleteJob from '../users/UserDeleteJob'
import UserPatchJob from '../users/UserPatchJob'
import WebhookJob from '../channels/webhook/WebhookJob'
import WebhookJob from '../providers/webhook/WebhookJob'
import { QueueConfig } from '../queue/Queue'
import JourneyDelayJob from '../journey/JourneyDelayJob'
import JourneyProcessJob from '../journey/JourneyProcessJob'

View file

@ -83,7 +83,7 @@ export default class Model {
}
static query<T extends typeof Model>(this: T, db: Database = App.main.db): Database.QueryBuilder<InstanceType<T>> {
return this.table(db).clearSelect()
return this.table(db)
}
static async first<T extends typeof Model>(
@ -91,7 +91,7 @@ export default class Model {
query: Query,
db: Database = App.main.db,
): Promise<InstanceType<T> | undefined> {
const record = await query(this.table(db)).first()
const record = await this.build(query, db).first()
if (!record) return undefined
return this.fromJson(record)
}
@ -107,7 +107,7 @@ export default class Model {
id = parseInt(id, 10)
if (isNaN(id)) return undefined
}
const record = await query(this.table(db))
const record = await this.build(query, db)
.where(`${this.tableName}.id`, id)
.first()
if (!record) return undefined
@ -119,7 +119,7 @@ export default class Model {
query: Query = qb => qb,
db: Database = App.main.db,
): Promise<InstanceType<T>[]> {
const records = await query(this.table(db))
const records = await this.build(query, db)
return records.map((item: any) => this.fromJson(item))
}
@ -145,7 +145,9 @@ export default class Model {
const total = await this.count(query, db)
const start = page * itemsPerPage
const results: T[] = total > 0
? await query(this.table(db)).offset(start).limit(itemsPerPage)
? await this.build(query, db)
.offset(start)
.limit(itemsPerPage)
: []
const end = Math.min(start + itemsPerPage, start + results.length)
return {
@ -210,7 +212,7 @@ export default class Model {
query: Query = qb => qb,
db: Database = App.main.db,
): Promise<Page<InstanceType<T>, B>> {
const records = await query(this.table(db))
const records = await this.build(query, db)
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
.where('id', '<', params.sinceId)
@ -277,10 +279,22 @@ export default class Model {
}
static table(db: Database = App.main.db): Database.QueryBuilder<any> {
return db(this.tableName).select(`${this.tableName}.*`)
return db(this.tableName)
}
static raw = raw
static build<T extends typeof Model>(
query: Query,
db: Database = App.main.db,
): Database.QueryBuilder<InstanceType<T>> {
const builder = query(this.table(db)) as any
const hasSelects = builder._statements.find((item: any) => item.grouping === 'columns')
if (!hasSelects) {
builder.select(`${this.tableName}.*`)
}
return builder
}
}
export type ModelParams = 'id' | 'created_at' | 'updated_at' | 'parseJson' | 'project_id' | 'toJSON'

View file

@ -1,7 +1,7 @@
import Model, { ModelParams } from '../core/Model'
import { JSONSchemaType } from '../core/validate'
export type ProviderGroup = 'email' | 'text' | 'push' | 'webhook'
export type ProviderGroup = 'email' | 'text' | 'push' | 'webhook' | 'analytics'
export interface ProviderMeta {
name: string
description?: string
@ -26,6 +26,10 @@ export const ProviderSchema = <T extends ExternalProviderParams, D>(id: string,
type: 'string',
nullable: true,
},
is_default: {
type: 'boolean',
nullable: true,
},
data,
},
additionalProperties: false,
@ -39,6 +43,7 @@ export default class Provider extends Model {
external_id?: string
group!: ProviderGroup
data!: Record<string, any>
is_default!: boolean
static jsonAttributes = ['data']
@ -52,18 +57,17 @@ export default class Provider extends Model {
static get cacheKey() {
return {
external(externalId: string) {
return `providers_external_${externalId}`
return `providers:external:${externalId}`
},
internal(id: number) {
return `providers_${id}`
return `providers:id:${id}`
},
default(projectId: number, group: string) {
return `providers:project:${projectId}:${group}`
},
}
}
static externalCacheKey(externalId: string) {
return `providers_${externalId}`
}
parseJson(json: any): void {
super.parseJson(json)

View file

@ -2,6 +2,7 @@ import Router from '@koa/router'
import { ProjectState } from '../auth/AuthMiddleware'
import { searchParamsSchema } from '../core/searchParams'
import { extractQueryParams } from '../utilities'
import { loadAnalyticsControllers } from './analytics'
import { loadEmailControllers } from './email'
import { ProviderMeta } from './Provider'
import { allProviders, pagedProviders } from './ProviderService'
@ -19,6 +20,7 @@ loadTextControllers(router, providers)
loadEmailControllers(router, providers)
loadWebhookControllers(router, providers)
loadPushControllers(router, providers)
loadAnalyticsControllers(router, providers)
router.get('/', async ctx => {
const params = extractQueryParams(ctx.query, searchParamsSchema)

View file

@ -24,6 +24,26 @@ export const loadProvider = async <T extends Provider>(id: number, projectId: nu
return mappedValue
}
export const loadDefaultProvider = async <T extends Provider>(group: string, projectId: number, mapper: ProviderMap<T>, app = App.main) => {
// Check if value is cached in memory
const cache = app.get<T>(Provider.cacheKey.default(projectId, group))
if (cache) return cache
// If not, fetch from DB
const record = await Provider.table()
.where('project_id', projectId)
.where('group', group)
.where('is_default', true)
.first()
if (!record) return
// Map to appropriate type, cache and return
const mappedValue = mapper(record)
cacheProvider(mappedValue)
return mappedValue
}
export const getProviderByExternalId = async <T extends Provider>(externalId: string, mapper: ProviderMap<T>, app = App.main): Promise<T | undefined> => {
// Check if value is cached in memory
@ -52,6 +72,7 @@ export const createProvider = async (projectId: number, params: ProviderParams)
export const updateProvider = async (id: number, params: ExternalProviderParams, app = App.main) => {
const provider = await Provider.updateAndFetch(id, params)
app.remove(Provider.cacheKey.internal(provider.id))
app.remove(Provider.cacheKey.default(provider.project_id, provider.group))
if (provider.external_id) {
app.remove(Provider.cacheKey.external(provider.external_id))
}
@ -60,6 +81,10 @@ export const updateProvider = async (id: number, params: ExternalProviderParams,
export const cacheProvider = (provider: Provider, app = App.main) => {
app.set(Provider.cacheKey.internal(provider.id), provider)
if (!provider.external_id) return
app.set(Provider.cacheKey.external(provider.external_id), provider)
if (provider.is_default) {
app.set(Provider.cacheKey.default(provider.project_id, provider.group), provider)
}
if (provider.external_id) {
app.set(Provider.cacheKey.external(provider.external_id), provider)
}
}

View file

@ -17,7 +17,7 @@ export const pagedProviders = async (params: SearchParams, projectId: number) =>
)
}
export const createController = <T extends ExternalProviderParams>(group: ProviderGroup, type: string, schema: JSONSchemaType<T>, externalKey: (payload: T) => string): Router => {
export const createController = <T extends ExternalProviderParams>(group: ProviderGroup, type: string, schema: JSONSchemaType<T>, externalKey?: (payload: T) => string): Router => {
const router = new Router<
ProjectState & { provider?: Provider }
>({
@ -27,7 +27,7 @@ export const createController = <T extends ExternalProviderParams>(group: Provid
router.post('/', async ctx => {
const payload = validate(schema, ctx.request.body)
ctx.body = await createProvider(ctx.state.project.id, { ...payload, external_id: externalKey(payload), type, group })
ctx.body = await createProvider(ctx.state.project.id, { ...payload, external_id: externalKey?.(payload), type, group })
})
router.param('providerId', async (value, ctx, next) => {

View file

@ -0,0 +1,21 @@
import { DriverConfig } from '../../config/env'
import { AnalyticsProvider, AnalyticsProviderName, AnalyticsUserEvent } from './AnalyticsProvider'
import { SegmentConfig } from './SegmentProvider'
export type AnalyticsConfig = SegmentConfig
export interface AnalyticsTypeConfig extends DriverConfig {
driver: AnalyticsProviderName
}
export default class Analytics {
readonly provider?: AnalyticsProvider
constructor(provider?: AnalyticsProvider) {
this.provider = provider
}
async track(event: AnalyticsUserEvent) {
await this.provider?.track(event)
}
}

View file

@ -0,0 +1,10 @@
import { UserEventParams } from '../../users/UserEvent'
import Provider from '../Provider'
export type AnalyticsProviderName = 'segment'
export type AnalyticsUserEvent = UserEventParams & { external_id: string }
export abstract class AnalyticsProvider extends Provider {
abstract track(event: AnalyticsUserEvent): Promise<void>
}

View file

@ -0,0 +1,58 @@
import Router from '@koa/router'
import { Analytics as Segment } from '@segment/analytics-node'
import { ProviderParams, ProviderSchema } from '../Provider'
import { AnalyticsTypeConfig } from './Analytics'
import { AnalyticsProvider, AnalyticsUserEvent } from './AnalyticsProvider'
import { createController } from '../ProviderService'
export interface SegmentConfig extends AnalyticsTypeConfig {
driver: 'segment'
writeKey: string
}
interface SegmentDataParams {
write_key: string
}
interface SegmentProviderParams extends ProviderParams {
data: SegmentDataParams
}
export default class SegmentAnalyticsProvider extends AnalyticsProvider {
write_key!: string
static namespace = 'segment'
static meta = {
name: 'Segment',
url: 'https://segment.com',
icon: 'https://parcelvoy.com/images/segment.svg',
}
static schema = ProviderSchema<SegmentProviderParams, SegmentDataParams>('segmentAnalyticsProviderParams', {
type: 'object',
required: ['write_key'],
properties: {
write_key: { type: 'string' },
},
})
segment!: Segment
async track(event: AnalyticsUserEvent) {
// If Segment has not been initialized yet, start it
if (!this.segment) {
this.segment = new Segment({ writeKey: this.write_key })
}
this.segment.track({
userId: event.external_id,
event: event.name,
properties: event.data,
})
}
static controllers(): Router {
return createController('analytics', this.namespace, this.schema)
}
}

View file

@ -0,0 +1,35 @@
import Router from '@koa/router'
import { ProviderMeta } from '../Provider'
import { loadDefaultProvider } from '../ProviderRepository'
import Analytics from './Analytics'
import { AnalyticsProvider, AnalyticsProviderName } from './AnalyticsProvider'
import SegmentAnalyticsProvider from './SegmentProvider'
const typeMap = {
segment: SegmentAnalyticsProvider,
}
export const providerMap = (record: { type: AnalyticsProviderName }): AnalyticsProvider => {
return typeMap[record.type].fromJson(record)
}
export const loadAnalytics = async (projectId: number): Promise<Analytics> => {
const provider = await loadDefaultProvider('analytics', projectId, providerMap)
return new Analytics(provider)
}
export const loadAnalyticsControllers = async (router: Router, providers: ProviderMeta[]) => {
for (const type of Object.values(typeMap)) {
const controllers = type.controllers()
router.use(
controllers.routes(),
controllers.allowedMethods(),
)
providers.push({
...type.meta,
type: type.namespace,
channel: 'analytics',
schema: type.schema,
})
}
}

View file

@ -35,9 +35,7 @@ export default class EmailJob extends Job {
await updateSendState(campaign, user)
// Create an event on the user about the email
await createEvent({
project_id: user.project_id,
user_id: user.id,
await createEvent(user, {
name: 'email_sent',
data: context,
})

View file

@ -38,9 +38,7 @@ export default class PushJob extends Job {
await updateSendState(campaign, user)
// Create an event on the user about the push
await createEvent({
project_id: user.project_id,
user_id: user.id,
await createEvent(user, {
name: 'push_sent',
data: context,
})
@ -56,9 +54,7 @@ export default class PushJob extends Job {
await updateSendState(campaign, user, 'failed')
// Create an event about the disabling
await createEvent({
project_id: user.project_id,
user_id: user.id,
await createEvent(user, {
name: 'notifications_disabled',
data: {
...context,

View file

@ -36,9 +36,7 @@ export default class TextJob extends Job {
await updateSendState(campaign, user)
// Create an event on the user about the text
createEvent({
project_id: user.project_id,
user_id: user.id,
await createEvent(user, {
name: 'text_sent',
data: context,
})

View file

@ -35,9 +35,7 @@ export default class WebhookJob extends Job {
await updateSendState(campaign, user)
// Create an event on the user about the email
createEvent({
project_id: user.project_id,
user_id: user.id,
createEvent(user, {
name: 'webhook_sent',
data: context,
})

View file

@ -1,5 +1,5 @@
import Render, { Variables } from '.'
import { Webhook } from '../channels/webhook/Webhook'
import { Webhook } from '../providers/webhook/Webhook'
import { ChannelType } from '../config/channels'
import Model, { ModelParams } from '../core/Model'
import { isValid, IsValidSchema } from '../core/validate'

View file

@ -1,6 +1,6 @@
import Router from '@koa/router'
import App from '../app'
import { loadTextChannelInbound } from '../channels/text'
import { loadTextChannelInbound } from '../providers/text'
import { RequestError } from '../core/errors'
import { JSONSchemaType, validate } from '../core/validate'
import Subscription, { SubscriptionParams } from './Subscription'

View file

@ -1,4 +1,4 @@
import { TextProvider } from '../channels/text/TextProvider'
import { TextProvider } from '../providers/text/TextProvider'
import { ChannelType } from '../config/channels'
import { SearchParams } from '../core/searchParams'
import { paramsToEncodedLink, TrackedLinkParams } from '../render/LinkService'
@ -104,9 +104,7 @@ export const toggleSubscription = async (userId: number, subscriptionId: number,
})
}
createEvent({
project_id: user.project_id,
user_id: user.id,
createEvent(user, {
name: state === SubscriptionState.unsubscribed
? 'unsubscribed'
: 'subscribed',

View file

@ -9,9 +9,9 @@ import { extractQueryParams } from '../utilities'
import { searchParamsSchema } from '../core/searchParams'
import { getUser, pagedUsers } from './UserRepository'
import { getUserLists } from '../lists/ListService'
import { getUserEvents } from './UserEventRepository'
import { getUserSubscriptions, toggleSubscription } from '../subscriptions/SubscriptionService'
import { SubscriptionState } from '../subscriptions/Subscription'
import { getUserEvents } from './UserEventRepository'
const router = new Router<
ProjectState & { user?: User }

View file

@ -1,4 +1,4 @@
import Model, { ModelParams } from '../core/Model'
import Model from '../core/Model'
export interface TemplateEvent extends Record<string, any> {
name: string
@ -20,4 +20,4 @@ export class UserEvent extends Model {
}
}
export type UserEventParams = Omit<UserEvent, ModelParams | 'flatten'> & { project_id: number }
export type UserEventParams = Pick<UserEvent, 'name' | 'data'>

View file

@ -1,8 +1,27 @@
import { SearchParams } from '../core/searchParams'
import { loadAnalytics } from '../providers/analytics'
import { User } from '../users/User'
import { UserEvent, UserEventParams } from './UserEvent'
export const createEvent = async (event: UserEventParams): Promise<UserEvent> => {
return await UserEvent.insertAndFetch(event)
export const createEvent = async (user: User, event: UserEventParams): Promise<number> => {
const data = {
project_id: user.project_id,
user_id: user.id,
...event,
}
const id = await UserEvent.insert(data)
const analytics = await loadAnalytics(user.project_id)
analytics.track({
external_id: user.external_id,
...event,
})
return id
}
export const createAndFetchEvent = async (user: User, event: UserEventParams): Promise<UserEvent> => {
const id = await createEvent(user, event)
const userEvent = await UserEvent.find(id)
return userEvent!
}
export const getUserEvents = async (id: number, params: SearchParams, projectId: number) => {

View file

@ -20,7 +20,7 @@ export default function Iframe({ content, fullHeight = true }: IframeProps) {
}
return (
<iframe src="about:blank" scrolling="no" frameBorder="0" ref={writeHTML}
<iframe src="about:blank" frameBorder="0" ref={writeHTML}
/>
)
}

View file

@ -8,11 +8,9 @@
border: 1px solid var(--color-grey);
}
.preview-window {
border-radius: var(--border-radius);
width: 100%;
min-height: calc(80vh - 80px);
border: 0;
.preview iframe {
overflow: scroll;
height: 500px;
}
.email-frame {

View file

@ -133,7 +133,6 @@
.source-preview, .source-preview iframe {
border-radius: var(--border-radius);
overflow: hidden;
}
.source-preview iframe {

94
package-lock.json generated
View file

@ -25,6 +25,7 @@
"@ladjs/country-language": "^1.0.3",
"@node-saml/node-saml": "github:node-saml/node-saml",
"@rxfork/sqs-consumer": "^6.0.0",
"@segment/analytics-node": "^1.0.0-beta.23",
"ajv": "^8.11.0",
"ajv-errors": "^3.0.0",
"ajv-formats": "^2.1.1",
@ -4690,6 +4691,25 @@
"node": "^14.15.0 || >=16.0.0"
}
},
"node_modules/@lukeed/csprng": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.0.1.tgz",
"integrity": "sha512-uSvJdwQU5nK+Vdf6zxcWAY2A8r7uqe+gePwLWzJ+fsQehq18pc0I2hJKwypZ2aLM90+Er9u1xn4iLJPZ+xlL4g==",
"engines": {
"node": ">=8"
}
},
"node_modules/@lukeed/uuid": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@lukeed/uuid/-/uuid-2.0.0.tgz",
"integrity": "sha512-dUz8OmYvlY5A9wXaroHIMSPASpSYRLCqbPvxGSyHguhtTQIy24lC+EGxQlwv71AhRCO55WOtgwhzQLpw27JaJQ==",
"dependencies": {
"@lukeed/csprng": "^1.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/@monaco-editor/loader": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@monaco-editor/loader/-/loader-1.3.2.tgz",
@ -6060,6 +6080,62 @@
"@aws-sdk/client-sqs": "^3.5.0"
}
},
"node_modules/@segment/analytics-core": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@segment/analytics-core/-/analytics-core-1.2.2.tgz",
"integrity": "sha512-zVWSDcyh7Rp32xL5v2fuEk2yZxxy+JA93vF1L3EF9XAYLSra/uEHJEswOWieXSdDHVRHes7APORp136usFE/tw==",
"dependencies": {
"@lukeed/uuid": "^2.0.0",
"dset": "^3.1.2",
"tslib": "^2.4.1"
}
},
"node_modules/@segment/analytics-node": {
"version": "1.0.0-beta.23",
"resolved": "https://registry.npmjs.org/@segment/analytics-node/-/analytics-node-1.0.0-beta.23.tgz",
"integrity": "sha512-S1P6opKo1WfBxzX2MeBqFmZ9ViMUmP3RpXNFqeCnkM2c7hqY3FEZJNV7CrOdH3ej5W1S8cUE2/U6dJgzt6jv3g==",
"dependencies": {
"@segment/analytics-core": "1.2.2",
"buffer": "^6.0.3",
"node-fetch": "^2.6.7",
"tslib": "^2.4.1",
"uuid": "^9.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@segment/analytics-node/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"node_modules/@segment/analytics-node/node_modules/uuid": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
"integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/@sinclair/typebox": {
"version": "0.24.51",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.51.tgz",
@ -11828,6 +11904,14 @@
"resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz",
"integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA=="
},
"node_modules/dset": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/dset/-/dset-3.1.2.tgz",
"integrity": "sha512-g/M9sqy3oHe477Ar4voQxWtaPIFw1jTdKZuomOjhCcBx9nHUNn0pu6NopuFFrTh/TRZIKEj+76vLWFu9BNKk+Q==",
"engines": {
"node": ">=4"
}
},
"node_modules/duplexer": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
@ -11910,7 +11994,6 @@
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
"integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
"dev": true,
"optional": true,
"dependencies": {
"iconv-lite": "^0.6.2"
@ -11920,7 +12003,6 @@
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
"dev": true,
"optional": true,
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
@ -20422,7 +20504,6 @@
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"dev": true,
"dependencies": {
"whatwg-url": "^5.0.0"
},
@ -28410,8 +28491,7 @@
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
"dev": true
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/treeverse": {
"version": "2.0.0",
@ -29241,8 +29321,7 @@
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
"dev": true
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/webpack": {
"version": "5.75.0",
@ -29493,7 +29572,6 @@
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dev": true,
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"