init

src/lib/server/auth.ts

@@ -0,0 +1,148 @@
+import type { RequestEvent } from '@sveltejs/kit'
+import { eq } from 'drizzle-orm'
+import { sha256 } from '@oslojs/crypto/sha2'
+import { encodeBase64url, encodeHexLowerCase } from '@oslojs/encoding'
+import { db } from '$lib/server/db'
+import {
+  deleteSession,
+  getSessionData,
+  renewSession,
+  type Session,
+} from './db/session'
+
+import { tables } from './db'
+import { Effect } from 'effect'
+
+const DAY_IN_MS = 1000 * 60 * 60 * 24
+
+export const sessionCookieName = 'auth-session'
+
+export function generateSessionToken() {
+  const bytes = crypto.getRandomValues(new Uint8Array(18))
+  const token = encodeBase64url(bytes)
+  return token
+}
+
+export async function createSession(token: string, userId: string) {
+  const sessionId = encodeHexLowerCase(sha256(new TextEncoder().encode(token)))
+  const session: Session = {
+    id: sessionId,
+    userId,
+    expiresAt: new Date(Date.now() + DAY_IN_MS * 30),
+  }
+  await db.insert(tables.session).values(session)
+  return session
+}
+
+export type SessionData =
+  | {
+      user: {
+        username: string
+        id: string
+      }
+      session: {
+        id: string
+        userId: string
+        expiresAt: Date
+      }
+    }
+  | {
+      user: null
+      session: null
+    }
+
+export const val = (token: string) =>
+  Effect.gen(function* () {
+    const sessionId = encodeHexLowerCase(
+      sha256(new TextEncoder().encode(token)),
+    )
+
+    const result: SessionData = { session: null, user: null }
+
+    const sessionData = yield* getSessionData(sessionId)
+
+    if (!sessionData) {
+      yield* Effect.logInfo(`Session ${sessionId} not found`)
+      return result
+    }
+
+    const { session, user } = sessionData
+
+    yield* Effect.logInfo('Session found', session)
+
+    const sessionExpired = Date.now() >= session.expiresAt.getTime()
+    if (sessionExpired) {
+      deleteSession(sessionId)
+      return result
+    }
+
+    yield* renewSession(session)
+
+    return { session, user }
+  }).pipe(Effect.withSpan('ValidateToken', { captureStackTrace: true }))
+
+export async function validateSessionToken(
+  token: string,
+): Promise<
+  | { session: null; user: null }
+  | { session: Session; user: { username: string; id: string } }
+> {
+  const sessionId = encodeHexLowerCase(sha256(new TextEncoder().encode(token)))
+  const [result] = await db
+    .select({
+      // Adjust user table here to tweak returned data
+      user: { id: tables.user.id, username: tables.user.username },
+      session: tables.session,
+    })
+    .from(tables.session)
+    .innerJoin(tables.user, eq(tables.session.userId, tables.user.id))
+    .where(eq(tables.session.id, sessionId))
+
+  if (!result) {
+    return { session: null, user: null }
+  }
+  const { session, user } = result
+
+  const sessionExpired = Date.now() >= session.expiresAt.getTime()
+  if (sessionExpired) {
+    await db.delete(tables.session).where(eq(tables.session.id, session.id))
+    return { session: null, user: null }
+  }
+
+  const renewSession =
+    Date.now() >= session.expiresAt.getTime() - DAY_IN_MS * 15
+  if (renewSession) {
+    session.expiresAt = new Date(Date.now() + DAY_IN_MS * 30)
+    await db
+      .update(tables.session)
+      .set({ expiresAt: session.expiresAt })
+      .where(eq(tables.session.id, session.id))
+  }
+
+  return { session, user }
+}
+
+export type SessionValidationResult = Awaited<
+  ReturnType<typeof validateSessionToken>
+>
+
+export async function invalidateSession(sessionId: string) {
+  await db.delete(tables.session).where(eq(tables.session.id, sessionId))
+}
+
+export function setSessionTokenCookie(
+  event: RequestEvent,
+  token: string,
+  expiresAt: Date,
+) {
+  event.cookies.set(sessionCookieName, token, {
+    expires: expiresAt,
+    path: '/',
+  })
+}
+
+export function deleteSessionTokenCookie(event: RequestEvent) {
+  event.cookies.delete(sessionCookieName, {
+    path: '/',
+  })
+}

src/lib/server/config.ts

@@ -0,0 +1,20 @@
+import { Config, Context, Effect } from 'effect'
+
+export class ConfigService extends Context.Tag('ConfigService')<
+	ConfigService,
+	{ readonly getDbUrl: () => Config.Config<URL> }
+>() {}
+
+const program = Effect.gen(function* () {
+	const configService = yield* ConfigService
+
+	const url = yield* configService.getDbUrl()
+
+	console.log(url.toString())
+})
+
+const runnable = Effect.provideService(program, ConfigService, {
+	getDbUrl: () => {
+		return Config.url('DATABASE_URL').pipe(Config.withDefault(new URL('file:local.db')))
+	}
+})

src/lib/server/crawler/clevertronik/data/capacity.ts

@@ -0,0 +1,13 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getCapacityVariations } from '../variations/capacity'
+import { getSelected } from '.'
+
+export const getCapacity = (page: Page) =>
+  Effect.gen(function* () {
+    const capacityVariations = yield* getCapacityVariations(page)
+
+    const selectedCapacity = yield* getSelected(capacityVariations)
+
+    return selectedCapacity.label
+  })

src/lib/server/crawler/clevertronik/data/color.ts

@@ -0,0 +1,13 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getSelected } from '.'
+import { getColorVariations } from '../variations/color'
+
+export const getColor = (page: Page) =>
+  Effect.gen(function* () {
+    const colorVariations = yield* getColorVariations(page)
+
+    const selectedCapacity = yield* getSelected(colorVariations)
+
+    return selectedCapacity.label
+  })

src/lib/server/crawler/clevertronik/data/condition.ts

@@ -0,0 +1,49 @@
+import { Data, Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { CrawlerElementNotFoundError } from '../..'
+
+const PRODUCT_CONDITION_SELECTOR =
+  '#dynamic_content > div.modell_page_grid > div.modell_page_name > div.modell_page_name_container > div'
+
+export const getCondition = (page: Page) =>
+  Effect.gen(function* () {
+    let conditionString = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate((PRODUCT_CONDITION_SELECTOR) => {
+          const productConditionDiv: HTMLDivElement | null =
+            document.querySelector(PRODUCT_CONDITION_SELECTOR)
+
+          if (!productConditionDiv) {
+            throw new Error('Could not find product condition')
+          }
+
+          return productConditionDiv.innerText
+        }, PRODUCT_CONDITION_SELECTOR),
+      catch: (cause) =>
+        new CrawlerElementNotFoundError({
+          message: 'Could not parse product condition',
+          cause,
+        }),
+    })
+
+    conditionString = conditionString.replace('Akku: Neu', '')
+
+    const [_, condition] = conditionString.split(':').map((str) => str.trim())
+
+    if (!condition) {
+      return yield* Effect.fail(
+        new ParseConditionError({
+          message: 'Could not parse condition from condition string',
+        }),
+      )
+    }
+
+    return condition
+  })
+
+export class ParseConditionError extends Data.TaggedError(
+  'ParseConditionError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}

src/lib/server/crawler/clevertronik/data/devices.ts

@@ -0,0 +1,130 @@
+import { Data, Effect } from 'effect'
+import type { Page } from 'puppeteer'
+
+const OPEN_MODAL_BUTTON_SELECTOR =
+  '#dynamic_content > div.modell_page_grid > div.modell_page_filter_and_info > div.show_single_container > div.show_single_text > button'
+
+const DEVICE_LIST_SELECTOR =
+  '#popup-singles > div.popup_box > div > div.single_item_list.js_single_item_list'
+
+export class ParseDeviceModalError extends Data.TaggedError(
+  'ParseDeviceModalError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export class OpenDeviceModalError extends Data.TaggedError(
+  'OpenDeviceModalError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+const openDeviceModal = (page: Page) =>
+  Effect.gen(function* () {
+    const childrenCount = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate(
+          async (DEVICE_LIST_SELECTOR, OPEN_MODAL_BUTTON_SELECTOR) => {
+            const button: HTMLButtonElement | null = document.querySelector(
+              OPEN_MODAL_BUTTON_SELECTOR,
+            )
+
+            if (!button) return
+
+            button.click()
+
+            let deviceListDiv: HTMLDivElement | null = null
+
+            const tryLimit = 200
+            for (let tryCount = 0; tryCount < tryLimit; tryCount++) {
+              deviceListDiv = document.querySelector(DEVICE_LIST_SELECTOR)
+
+              if (deviceListDiv) {
+                break
+              }
+              await new Promise((resolve) => setTimeout(resolve, 50))
+            }
+
+            if (!deviceListDiv) {
+              throw new Error('List not found')
+            }
+
+            return deviceListDiv.children.length
+          },
+          DEVICE_LIST_SELECTOR,
+          OPEN_MODAL_BUTTON_SELECTOR,
+        ),
+      catch: (cause) =>
+        new OpenDeviceModalError({
+          message: `Unable to open device modal: ${cause}`,
+          cause,
+        }),
+    })
+  })
+
+export const getDevices = (page: Page) =>
+  Effect.gen(function* () {
+    yield* openDeviceModal(page)
+
+    return yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate((DEVICE_LIST_SELECTOR) => {
+          const deviceListDiv: HTMLDivElement | null =
+            document.querySelector(DEVICE_LIST_SELECTOR)
+
+          if (!deviceListDiv) {
+            throw new Error('Unable to find device list')
+          }
+
+          const parseDevice = (element: HTMLDivElement) => {
+            const detailListElements: HTMLLIElement[] = Array.from(
+              element.querySelectorAll('.inline_list'),
+            )
+
+            const details = detailListElements.map((li) => li.innerText)
+
+            const extraBarDiv: HTMLDivElement | null = element.querySelector(
+              '.extra_information_list',
+            )
+
+            let batteryStatusString: string = ''
+            let imagesCountString: string = ''
+            let originalPagacking: string = ''
+
+            if (extraBarDiv) {
+              ;[batteryStatusString, imagesCountString, originalPagacking] =
+                extraBarDiv.innerText.split('\n')
+            }
+
+            let imageCount = 0
+
+            if (imagesCountString) {
+              imageCount = parseInt(imagesCountString.match(/\d+/)?.[0] ?? '0')
+            }
+
+            let batteryPercentage = 0
+
+            if (batteryStatusString) {
+              batteryPercentage = parseInt(
+                batteryStatusString.match(/\d+(?=%)/)?.[0] ?? '0',
+              )
+            }
+
+            return { details, originalPagacking, imageCount, batteryPercentage }
+          }
+
+          const deviceDivs: HTMLDivElement[] = Array.from(
+            deviceListDiv.querySelectorAll(':scope > div'),
+          )
+
+          return deviceDivs.map((device) => parseDevice(device))
+        }, DEVICE_LIST_SELECTOR),
+      catch: (cause) =>
+        new ParseDeviceModalError({
+          message: `Unable to parse device list: ${cause}`,
+          cause,
+        }),
+    })
+  })

src/lib/server/crawler/clevertronik/data/index.ts

@@ -0,0 +1,93 @@
+import { Data, Effect } from 'effect'
+import { getPrice } from './price'
+import type { Page } from 'puppeteer'
+import { getProductName } from './productName'
+import { getCondition } from './condition'
+import { getCapacity } from './capacity'
+import type { Variation } from '../variations'
+import { getColor } from './color'
+import { getSim } from './sim'
+import { getStockLevel } from './stockLevel'
+import { getDevices } from './devices'
+
+export class ExtractSelectedVariationError extends Data.TaggedError(
+  'ExtractSelectedVariationError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export const getSelected = (variations: Variation[]) =>
+  Effect.gen(function* () {
+    const selected = variations.find((variation) => variation.selected)
+
+    if (!selected) {
+      return yield* Effect.fail(
+        new ExtractSelectedVariationError({
+          message: 'No selected variation option found',
+        }),
+      )
+    }
+
+    return selected
+  })
+
+const log = (value: unknown, name: string) =>
+  Effect.gen(function* () {
+    if (typeof value === 'object') {
+      yield* Effect.logInfo(`${name}: ${JSON.stringify(value)}`)
+    } else {
+      yield* Effect.logInfo(`${name}: ${value}`)
+    }
+  })
+
+export type PageData = {
+  price: number
+  productName: string
+  condition: string
+  capacity: string
+  color: string
+  sim: string
+  stockLevel: string
+  devices: DeviceData[]
+}
+
+export type DeviceData = {
+  details: string[]
+  originalPagacking: string
+  imageCount: number
+  batteryPercentage: number
+}
+
+export const getPageData = (page: Page) =>
+  Effect.gen(function* () {
+    const price = yield* getPrice(page)
+
+    const productName = yield* getProductName(page).pipe(
+      Effect.tap((val) => log(val, 'Product Name')),
+    )
+
+    const condition = yield* getCondition(page)
+
+    const capacity = yield* getCapacity(page)
+
+    const color = yield* getColor(page)
+
+    const sim = yield* getSim(page)
+
+    const stockLevel = yield* getStockLevel(page)
+    const devices = yield* getDevices(page)
+
+    const pageData: PageData = {
+      price,
+      productName,
+      condition,
+      capacity,
+      color,
+      sim,
+      stockLevel,
+      devices,
+    }
+
+    return pageData
+  })

src/lib/server/crawler/clevertronik/data/price.ts

@@ -0,0 +1,57 @@
+import { Data, Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { CrawlerElementNotFoundError } from '../..'
+
+export class ParsePriceError extends Data.TaggedError('ParsePriceError')<{
+  cause?: unknown
+  message?: string
+}> {}
+
+const PRICE_SELECTOR =
+  '#js-buybox-container > div > div.buybox_price_container > div > span.js_adjust_main_price'
+
+export const getPrice = (page: Page) =>
+  Effect.gen(function* () {
+    const priceString = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate((PRICE_SELECTOR) => {
+          const priceSpan: HTMLSpanElement | null =
+            document.querySelector(PRICE_SELECTOR)
+
+          if (!priceSpan) {
+            throw new Error('Could not find price span')
+          }
+
+          return priceSpan.innerText
+        }, PRICE_SELECTOR),
+      catch: (cause) =>
+        new CrawlerElementNotFoundError({
+          message: 'Could not parse price',
+          cause,
+        }),
+    })
+
+    return priceString
+  }).pipe(Effect.flatMap(toNumber))
+
+const toNumber = (priceString: string) =>
+  Effect.gen(function* () {
+    // Remove all non-digit characters except comma and dot
+    let cleaned = priceString.replace(/[^\d,.-]/g, '')
+
+    // Replace comma with dot (for European format)
+    cleaned = cleaned.replace(',', '.')
+
+    const parsed = parseFloat(cleaned)
+
+    if (Number.isNaN(parsed)) {
+      return yield* Effect.fail(
+        new ParsePriceError({
+          message: 'Could not parse price string',
+          cause: 'Number is NaN',
+        }),
+      )
+    } else {
+      return yield* Effect.succeed(parsed)
+    }
+  })

src/lib/server/crawler/clevertronik/data/productName.ts

@@ -0,0 +1,28 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { CrawlerElementNotFoundError } from '../..'
+
+const PRODUCT_NAME_SELECTOR =
+  '#dynamic_content > div.modell_page_grid > div.modell_page_name > div.modell_page_name_container > h1 > span'
+
+export const getProductName = (page: Page) =>
+  Effect.gen(function* () {
+    return yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate((PRODUCT_NAME_SELECTOR) => {
+          const productNameSpan: HTMLSpanElement | null =
+            document.querySelector(PRODUCT_NAME_SELECTOR)
+
+          if (!productNameSpan) {
+            throw new Error('Could not find product name')
+          }
+
+          return productNameSpan.innerText
+        }, PRODUCT_NAME_SELECTOR),
+      catch: (cause) =>
+        new CrawlerElementNotFoundError({
+          message: 'Could not parse product name',
+          cause,
+        }),
+    })
+  })

src/lib/server/crawler/clevertronik/data/sim.ts

@@ -0,0 +1,17 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getSelected } from '.'
+import { getSimVariations } from '../variations/sim'
+
+export const getSim = (page: Page) =>
+  Effect.gen(function* () {
+    const simVariations = yield* getSimVariations(page)
+
+    if (simVariations.length === 0) {
+      return ''
+    }
+
+    const selectedSim = yield* getSelected(simVariations)
+
+    return selectedSim.label
+  })

src/lib/server/crawler/clevertronik/data/stockLevel.ts

@@ -0,0 +1,29 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { CrawlerElementNotFoundError } from '../..'
+
+const STOCK_LEVEL_BADGE_SELECTOR =
+  '.modellfilter_row > .modellfilter_head > .colored_label_wrapper'
+
+export const getStockLevel = (page: Page) =>
+  Effect.gen(function* () {
+    return yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate((STOCK_LEVEL_BADGE_SELECTOR) => {
+          const stockLevelBadge: HTMLDivElement | null = document.querySelector(
+            STOCK_LEVEL_BADGE_SELECTOR,
+          )
+
+          if (!stockLevelBadge) {
+            throw new Error('Could not find stock level')
+          }
+
+          return stockLevelBadge.innerText
+        }, STOCK_LEVEL_BADGE_SELECTOR),
+      catch: (cause) =>
+        new CrawlerElementNotFoundError({
+          message: `Could not parse stock level, ${cause}`,
+          cause,
+        }),
+    })
+  })

src/lib/server/crawler/clevertronik/index.ts

@@ -0,0 +1,80 @@
+import { Data, Effect, pipe } from 'effect'
+import { openPage } from '../openPage'
+import * as Crawler from '../index'
+import { withOperation } from '$lib/server/utils'
+import { getProductName } from './data/productName'
+import { getAllVariations, VariationEnum, type Variation } from './variations'
+import { getPageData, type PageData } from './data'
+import type { Page } from 'puppeteer'
+import {
+  getAllVariationCategoryData,
+  getAllVariationData,
+  type VariationCategory,
+} from './variationData'
+
+export type Selectable = {
+  selector: string
+}
+
+export class VariationParseError extends Data.TaggedError(
+  'VariationParseError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export const crawlClevertronik = Effect.gen(function* () {
+  const page = yield* openPage(
+    'https://www.clevertronic.de/kaufen/handy-kaufen/apple/iphone-16-pro-max/auswahl/21872/apple-iphone-16-pro-max-256gb-titan-schwarz?ok=11&ok_click=11',
+  )
+
+  yield* Effect.logInfo('opened clevertronik page')
+
+  let productName = yield* getProductName(page)
+
+  const variations = yield* getAllVariations(page)
+
+  // const variationCategory: VariationCategory = {
+  //   name: 'Capacity',
+  //   variations: variations.capacities,
+  //   subVariationCategory: {
+  //     name: 'Color',
+  //     variations: variations.colors,
+  //     subVariationCategory: {
+  //       name: 'Sim',
+  //       variations: variations.sims,
+  //       subVariationCategory: {
+  //         name: 'Condition',
+  //         variations: variations.conditions,
+  //         subVariationCategory: {
+  //           name: 'Battery',
+  //           variations: variations.battery,
+  //         },
+  //       },
+  //     },
+  //   },
+  // }
+
+  const variationCategory: VariationCategory = {
+    name: VariationEnum.CAPACITY,
+    subVariationCategory: {
+      name: VariationEnum.COLOR,
+    },
+  }
+
+  const variationData = yield* getAllVariationCategoryData(
+    page,
+    variationCategory,
+  )
+
+  yield* Effect.promise(() => page.browser().close())
+
+  return {
+    productName,
+    ...variations,
+    variationData,
+  }
+}).pipe(
+  Effect.provide(Crawler.CrawlerLayer()),
+  withOperation('CrawlClevertronik'),
+)

src/lib/server/crawler/clevertronik/variationData/index.ts

@@ -0,0 +1,138 @@
+import type { Page } from 'puppeteer'
+import { getPageData, type PageData } from '../data'
+import {
+  getVariationHandler,
+  VariationEnum,
+  VariationParseError,
+  type Variation,
+} from '../variations'
+import { Effect, pipe } from 'effect'
+
+export type VariationPageData = {
+  capacityVartiationName: string
+  conditionVartiationName: string
+  colorVartiationName: string
+  batteryVartiationName: string
+} & PageData
+
+export type VariationCategory = {
+  name: VariationEnum
+  subVariationCategory?: VariationCategory
+}
+
+export const getAllVariationCategoryData = (
+  page: Page,
+  variationCategory: VariationCategory,
+): Effect.Effect<
+  PageData[],
+  Effect.Effect.Error<ReturnType<typeof getDataFromVariation>>,
+  Effect.Effect.Context<ReturnType<typeof getDataFromVariation>>
+> =>
+  Effect.gen(function* () {
+    const result: PageData[] = []
+    const variationPage = yield* duplicatePage(page)
+    const variations =
+      yield* getVariationHandler[variationCategory.name](variationPage)
+
+    for (const variation of variations.filter((v) => v.active)) {
+      if (!variation.selected) {
+        yield* gotoVariation(variationPage, variation)
+      }
+      if (variationCategory.subVariationCategory) {
+        const pageDataResult = yield* getAllVariationCategoryData(
+          variationPage,
+          variationCategory.subVariationCategory,
+        )
+
+        result.push(...pageDataResult)
+      } else {
+        result.push(yield* getPageData(variationPage))
+      }
+    }
+
+    return result
+  })
+
+export const getAllVariationData = (page: Page, variations: Variation[]) =>
+  Effect.gen(function* () {
+    const effects = []
+    for (const variation of variations) {
+      if (!variation.active) {
+        continue
+      }
+      effects.push(getDataFromVariation(page, variation))
+    }
+
+    return yield* Effect.all(effects, { concurrency: 'unbounded' })
+  })
+
+const getDataFromVariation = (page: Page, variation: Variation) =>
+  pipe(
+    gotoVariation(page, variation),
+    Effect.flatMap(() => getPageData(page)),
+    Effect.annotateLogs({ variation: variation.label }),
+  )
+
+const duplicatePage = (page: Page) =>
+  Effect.gen(function* () {
+    const variationPage = yield* Effect.promise(() => page.browser().newPage())
+
+    yield* Effect.promise(() => variationPage.goto(page.url()))
+
+    return variationPage
+  })
+
+const gotoVariation = (page: Page, variation: Variation) =>
+  Effect.gen(function* () {
+    const success = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate(async (selector) => {
+          const button: HTMLButtonElement | null =
+            document.querySelector(selector)
+
+          const loadingDiv: HTMLDivElement | null = document.querySelector(
+            '#js-global-wait-mask',
+          )
+
+          if (!button) {
+            throw new Error('Could not find variation button')
+          }
+
+          if (!loadingDiv) {
+            throw new Error('Could not find loading div')
+          }
+
+          button.click()
+
+          let wasActive = false
+
+          const tryLimit = 200
+          for (let tryCount = 0; tryCount < tryLimit; tryCount++) {
+            await new Promise((resolve) => setTimeout(resolve, 50))
+
+            const current = loadingDiv.classList.contains('js_active')
+
+            if (!wasActive && current) {
+              wasActive = true
+              continue
+            }
+
+            if (wasActive && !current) {
+              return true
+            }
+          }
+          return false
+        }, variation.selector),
+      catch: (cause) =>
+        new VariationParseError({
+          message: `Could not goto variation: ${cause}`,
+          cause,
+        }),
+    })
+
+    yield* Effect.sleep('2 second')
+
+    yield* Effect.logInfo(`Goto Variaton success: ${success}`)
+
+    return page
+  })

src/lib/server/crawler/clevertronik/variations/battery.ts

@@ -0,0 +1,10 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getVariations } from '.'
+
+const BATTERY_CONTAINER_SELECTOR = '.new_battery_options'
+
+export const getBatteryVariations = (page: Page) =>
+  Effect.gen(function* () {
+    return yield* getVariations(page, BATTERY_CONTAINER_SELECTOR, 'DIV')
+  })

src/lib/server/crawler/clevertronik/variations/capacity.ts

@@ -0,0 +1,10 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getVariations } from '.'
+
+const CAPACITY_CONTAINER_SELECTOR = '[data-scrollel="prop_19"]'
+
+export const getCapacityVariations = (page: Page) =>
+  Effect.gen(function* () {
+    return yield* getVariations(page, CAPACITY_CONTAINER_SELECTOR)
+  })

src/lib/server/crawler/clevertronik/variations/color.ts

@@ -0,0 +1,23 @@
+import { Effect, pipe } from 'effect'
+import type { Page } from 'puppeteer'
+import { getVariations } from '.'
+
+const COLOR_VARIATION_CONTAINER_SELECTOR = '[data-scrollel="prop_17"]'
+
+export const getColorVariations = (page: Page) =>
+  pipe(
+    Effect.gen(function* () {
+      return yield* getVariations(page, COLOR_VARIATION_CONTAINER_SELECTOR)
+    }),
+    Effect.flatMap((variations) =>
+      Effect.gen(function* () {
+        for (const variation of variations) {
+          if (variation.detail?.includes('Alternativen')) {
+            variation.active = false
+          }
+        }
+
+        return yield* Effect.succeed(variations)
+      }),
+    ),
+  )

src/lib/server/crawler/clevertronik/variations/condition.ts

@@ -0,0 +1,10 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getVariations } from '.'
+
+export const CONDITION_CONTAINER_SELECTOR = '[data-scrollel="condition"]'
+
+export const getConditionVariations = (page: Page) =>
+  Effect.gen(function* () {
+    return yield* getVariations(page, CONDITION_CONTAINER_SELECTOR, 'DIV')
+  })

src/lib/server/crawler/clevertronik/variations/index.ts

@@ -0,0 +1,118 @@
+import { Data, Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import type { Selectable } from '../'
+import { getCapacityVariations } from './capacity'
+import { getColorVariations } from './color'
+import { getSimVariations } from './sim'
+import { getConditionVariations } from './condition'
+import { getBatteryVariations } from './battery'
+
+export type Variation = {
+  label: string
+  detail?: string
+  /**
+   * active and clickable
+   */
+  active: boolean
+  /**
+   * currently selected
+   */
+  selected: boolean
+} & Selectable
+
+export class VariationParseError extends Data.TaggedError(
+  'VariationParseError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export enum VariationEnum {
+  CAPACITY = 'Capacity',
+  COLOR = 'Color',
+  SIM = 'Sim',
+  CONDITION = 'CONDITION',
+  BATTERY = 'BATTER',
+}
+
+export const getVariationHandler = {
+  [VariationEnum.CAPACITY]: getCapacityVariations,
+  [VariationEnum.COLOR]: getColorVariations,
+  [VariationEnum.SIM]: getSimVariations,
+  [VariationEnum.CONDITION]: getConditionVariations,
+  [VariationEnum.BATTERY]: getBatteryVariations,
+}
+
+export const getVariations = (
+  page: Page,
+  containerSelector: string,
+  elementType: 'DIV' | 'SPAN' = 'SPAN',
+) =>
+  Effect.gen(function* () {
+    const variations = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate(
+          async (containerSelector, elementType) => {
+            const container = document.querySelector(containerSelector)
+
+            if (!container) {
+              throw new Error('Container not found')
+            }
+
+            const buttons: HTMLSpanElement[] | HTMLDivElement[] = Array.from(
+              container.children,
+            ).filter(
+              (child) => child.tagName === elementType,
+            ) as HTMLSpanElement[]
+
+            const variations: Variation[] = []
+
+            for (let i = 0; i < buttons.length; i++) {
+              const selector = `${containerSelector} > :nth-child(${i + 1})`
+              const textContent = buttons[i].innerText.split('\n')
+              variations.push({
+                label: textContent.at(0)?.trim() ?? '',
+                active: !buttons[i].classList.contains('faded'),
+                selected: buttons[i].classList.contains('active'),
+                detail: textContent.at(1)?.trim(),
+                selector,
+              })
+            }
+            return variations
+          },
+          containerSelector,
+          elementType,
+        ),
+      catch: (cause) =>
+        new VariationParseError({
+          message: `Could not parse variations, ${cause}, ${containerSelector}`,
+          cause,
+        }),
+    })
+
+    return variations
+  })
+
+export const getAllVariations = (page: Page) =>
+  Effect.gen(function* () {
+    const [capacities, colors, sims, conditions, battery] = yield* Effect.all(
+      [
+        getCapacityVariations(page),
+        getColorVariations(page),
+        getSimVariations(page),
+        getConditionVariations(page),
+        getBatteryVariations(page),
+      ],
+      {
+        concurrency: 'unbounded',
+      },
+    )
+
+    return {
+      capacities,
+      colors,
+      sims,
+      conditions,
+      battery,
+    }
+  })

src/lib/server/crawler/clevertronik/variations/sim.ts

@@ -0,0 +1,17 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getVariations, type Variation } from '.'
+
+const SIM_CONTAINER_SELECTOR = '[data-scrollel="prop_39"]'
+
+export const getSimVariations = (page: Page) =>
+  Effect.gen(function* () {
+    return yield* getVariations(page, SIM_CONTAINER_SELECTOR).pipe(
+      Effect.catchTag('VariationParseError', (e) =>
+        Effect.gen(function* () {
+          yield* Effect.logInfo('No SimVariations found')
+          return yield* Effect.succeed(new Array<Variation>())
+        }),
+      ),
+    )
+  }).pipe()

src/lib/server/crawler/index.ts

@@ -0,0 +1,82 @@
+import { Context, Data, Effect, Layer } from 'effect'
+import { launch } from 'puppeteer'
+
+export class CrawlerPageNotFoundError extends Data.TaggedError(
+  'CrawlerPageNotFoundError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export class CrawlerHttpError extends Data.TaggedError('CrawlerHttpError')<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export class CrawlerError extends Data.TaggedError('CrawlerError')<{
+  cause?: unknown
+  message?: string
+}> {}
+
+export class CrawlerElementNotFoundError extends Data.TaggedError(
+  'CrawlerElementNotFoundError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
+const getBrowser = (options?: Parameters<typeof launch>[0]) =>
+  Effect.acquireRelease(
+    Effect.tryPromise({
+      try: () =>
+        launch(options).then((res) => {
+          return res
+        }),
+      catch: (e) =>
+        new CrawlerError({ cause: e, message: 'Error launching browser' }),
+    }),
+    (browser) => Effect.promise(() => browser.close()),
+  )
+
+interface CrawlerImpl {
+  use: <T>(
+    fn: (client: Awaited<ReturnType<typeof launch>>) => T,
+  ) => Effect.Effect<Awaited<T>, CrawlerError, never>
+}
+
+export class Crawler extends Context.Tag('Crawler')<Crawler, CrawlerImpl>() {}
+
+export const make = (options?: Parameters<typeof launch>[0]) =>
+  Effect.gen(function* () {
+    const browser = yield* getBrowser(options)
+
+    return Crawler.of({
+      use: (fn) =>
+        Effect.gen(function* () {
+          const result = yield* Effect.try({
+            try: () => fn(browser),
+            catch: (e) => {
+              return new CrawlerError({
+                cause: e,
+                message: `Crawler error: ${e}`,
+              })
+            },
+          })
+          if (result instanceof Promise) {
+            return yield* Effect.tryPromise({
+              try: () => result,
+              catch: (e) =>
+                new CrawlerError({
+                  cause: e,
+                  message: 'Asyncronous error in `Crawler.use`',
+                }),
+            })
+          } else {
+            return result
+          }
+        }),
+    })
+  })
+
+export const CrawlerLayer = (options?: Parameters<typeof launch>[0]) =>
+  Layer.scoped(Crawler, make(options))

src/lib/server/crawler/openPage.ts

@@ -0,0 +1,46 @@
+import { Effect } from 'effect'
+import {
+  Crawler,
+  CrawlerError,
+  CrawlerHttpError,
+  CrawlerPageNotFoundError,
+} from './index'
+
+export const openPage = (url: string) =>
+  Effect.gen(function* () {
+    const browser = yield* Crawler
+
+    const page = yield* browser.use((b) =>
+      Effect.gen(function* () {
+        const page = yield* Effect.promise(() => b.newPage())
+
+        const res = yield* Effect.tryPromise({
+          try: () => page.goto(url),
+          catch: (e) =>
+            new CrawlerError({
+              message: `Error opening page: ${url}`,
+              cause: e,
+            }),
+        })
+
+        if (!res) {
+          return yield* Effect.fail(
+            new CrawlerPageNotFoundError({ message: 'Page not found' }),
+          )
+        }
+
+        if (res.ok()) {
+          return page
+        }
+
+        return yield* Effect.fail(
+          new CrawlerHttpError({
+            message: res.statusText(),
+            cause: res.status(),
+          }),
+        )
+      }),
+    )
+
+    return yield* page
+  })

src/lib/server/db/DB.ts

@@ -0,0 +1,81 @@
+import { drizzle } from 'drizzle-orm/libsql'
+
+import { Config, Context, Data, Effect, Layer } from 'effect'
+import { createClient } from '@libsql/client'
+import { user } from './user'
+import { session } from './session'
+
+export const tables = { user, session }
+
+export class DbError extends Data.TaggedError('DbError')<{
+	cause?: unknown
+	message?: string
+}> {}
+
+export class DbOptimisticLockingError extends Data.TaggedError('DbOptimisticLockingError')<{
+	cause?: unknown
+	message?: string
+}> {}
+
+interface DbImpl {
+	use: <T>(
+		fn: (client: ReturnType<typeof drizzle>) => T,
+	) => Effect.Effect<Awaited<T>, DbError, never>
+	insert: (input: number) => Effect.Effect<void, DbError | DbOptimisticLockingError, never>
+}
+export class Db extends Context.Tag('Db')<Db, DbImpl>() {}
+
+export const make = (options: Parameters<typeof createClient>[0]) =>
+	Effect.gen(function* () {
+		const client = yield* Effect.acquireRelease(
+			Effect.try({
+				try: () => drizzle(createClient(options), { schema: tables }),
+				catch: (e) => new DbError({ cause: e, message: 'Error connecting' }),
+			}),
+			() => Effect.sync(() => Effect.logInfo('Disconnect Db')),
+		)
+		return Db.of({
+			insert: (n: number) => {
+				if (n === 0) {
+					throw new DbOptimisticLockingError({ message: 'OptimisticLocking failed with version 1' })
+				}
+
+				return Effect.succeed(undefined)
+			},
+			use: (fn) =>
+				Effect.gen(function* () {
+					const result = yield* Effect.try({
+						try: () => fn(client),
+						catch: (e) => {
+							return new DbError({
+								cause: e,
+								message: `DB error: ${e}`,
+							})
+						},
+					})
+					if (result instanceof Promise) {
+						return yield* Effect.tryPromise({
+							try: () => result,
+							catch: (e) =>
+								new DbError({
+									cause: e,
+									message: 'Asyncronous error in `Db.use`',
+								}),
+						})
+					} else {
+						return result
+					}
+				}),
+		})
+	})
+
+export const layer = (options: Parameters<typeof createClient>[0]) =>
+	Layer.scoped(Db, make(options))
+
+export const fromEnv = Layer.scoped(
+	Db,
+	Effect.gen(function* () {
+		const url = yield* Config.string('DATABASE_URL')
+		return yield* make({ url })
+	}),
+)

src/lib/server/db/index.ts

@@ -0,0 +1,73 @@
+import { drizzle } from 'drizzle-orm/libsql'
+
+import { env } from '$env/dynamic/private'
+
+import { Config, Context, Data, Effect, Layer } from 'effect'
+import { createClient } from '@libsql/client'
+import { user } from './user'
+import { session } from './session'
+
+export const tables = { user, session }
+
+export class DbError extends Data.TaggedError('DbError')<{
+	cause?: unknown
+	message?: string
+}> {}
+
+interface DbImpl {
+	use: <T>(
+		fn: (client: ReturnType<typeof drizzle>) => T
+	) => Effect.Effect<Awaited<T>, DbError, never>
+}
+export class Db extends Context.Tag('Db')<Db, DbImpl>() {}
+
+export const make = (options: Parameters<typeof createClient>[0]) =>
+	Effect.gen(function* () {
+		const client = yield* Effect.acquireRelease(
+			Effect.try({
+				try: () => drizzle(createClient(options), { schema: tables }),
+				catch: (e) => new DbError({ cause: e, message: 'Error connecting' })
+			}),
+			() => Effect.sync(() => Effect.logInfo('Disconnect Db'))
+		)
+		return Db.of({
+			use: (fn) =>
+				Effect.gen(function* () {
+					const result = yield* Effect.try({
+						try: () => fn(client),
+						catch: (e) =>
+							new DbError({
+								cause: e,
+								message: 'Syncronous error in `Db.use`'
+							})
+					})
+					if (result instanceof Promise) {
+						return yield* Effect.tryPromise({
+							try: () => result,
+							catch: (e) =>
+								new DbError({
+									cause: e,
+									message: 'Asyncronous error in `Db.use`'
+								})
+						})
+					} else {
+						return result
+					}
+				})
+		})
+	})
+
+export const layer = (options: Parameters<typeof createClient>[0]) =>
+	Layer.scoped(Db, make(options))
+
+export const fromEnv = Layer.scoped(
+	Db,
+	Effect.gen(function* () {
+		const url = yield* Config.string('DATABASE_URL')
+		return yield* make({ url })
+	})
+)
+
+const client = createClient({ url: env.DATABASE_URL })
+
+export const db = drizzle(client, { schema: tables })

src/lib/server/db/session.ts

@@ -0,0 +1,58 @@
+import { integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'
+import { eq } from 'drizzle-orm'
+import { user } from './user'
+import { Duration, Effect } from 'effect'
+
+import * as DB from './DB'
+
+export const session = sqliteTable('session', {
+	id: text('id').primaryKey(),
+	userId: text('user_id')
+		.notNull()
+		.references(() => user.id),
+	expiresAt: integer('expires_at', { mode: 'timestamp' }).notNull(),
+})
+
+export type Session = typeof session.$inferSelect
+
+export const getSessionData = (sessionId: string) =>
+	Effect.gen(function* () {
+		const db = yield* DB.Db
+
+		const result = yield* db.use(
+			async (db) =>
+				await db
+					.select({
+						user: { id: user.id, username: user.username },
+						session: session,
+					})
+					.from(session)
+					.innerJoin(user, eq(session.userId, user.id))
+					.where(eq(session.id, sessionId)),
+		)
+		return result.at(0)
+	}).pipe(Effect.provide(DB.fromEnv))
+
+export const deleteSession = (sessionId: string) =>
+	Effect.gen(function* () {
+		const db = yield* DB.Db
+		yield* db.use((db) => db.delete(DB.tables.session).where(eq(DB.tables.session.id, sessionId)))
+	}).pipe(Effect.provide(DB.fromEnv))
+
+export const renewSession = (session: Session) =>
+	Effect.gen(function* () {
+		const db = yield* DB.Db
+
+		const DAY_IN_MS = Duration.toMillis(Duration.days(1))
+
+		const renewSession = Date.now() >= session.expiresAt.getTime() - DAY_IN_MS * 15
+		if (renewSession) {
+			session.expiresAt = new Date(Date.now() + DAY_IN_MS * 30)
+			yield* db.use((db) =>
+				db
+					.update(DB.tables.session)
+					.set({ expiresAt: session.expiresAt })
+					.where(eq(DB.tables.session.id, session.id)),
+			)
+		}
+	}).pipe(Effect.provide(DB.fromEnv))

src/lib/server/db/user.ts

@@ -0,0 +1,10 @@
+import { integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'
+
+export const user = sqliteTable('user', {
+	id: text('id').primaryKey(),
+	age: integer('age'),
+	username: text('username').notNull().unique(),
+	passwordHash: text('password_hash').notNull()
+})
+
+export type User = typeof user.$inferSelect

src/lib/server/utils.ts

@@ -0,0 +1,4 @@
+import { Effect } from 'effect'
+
+export const withOperation = (operationName: string) =>
+  Effect.annotateLogs({ operation: operationName })