diff --git a/src/lib/server/crawler/clevertronik/data/battery.ts b/src/lib/server/crawler/clevertronik/data/battery.ts new file mode 100644 index 0000000..03c9e90 --- /dev/null +++ b/src/lib/server/crawler/clevertronik/data/battery.ts @@ -0,0 +1,13 @@ +import { Effect } from 'effect' +import type { Page } from 'puppeteer' +import { getSelected } from '.' +import { getBatteryVariations } from '../variations/battery' + +export const getBattery = (page: Page) => + Effect.gen(function* () { + const batteryVariations = yield* getBatteryVariations(page) + + const selectedCapacity = yield* getSelected(batteryVariations ?? []) + + return selectedCapacity.label + }) diff --git a/src/lib/server/crawler/clevertronik/data/index.ts b/src/lib/server/crawler/clevertronik/data/index.ts index ce3c7c5..001121e 100644 --- a/src/lib/server/crawler/clevertronik/data/index.ts +++ b/src/lib/server/crawler/clevertronik/data/index.ts @@ -9,6 +9,7 @@ import { getColor } from './color' import { getSim } from './sim' import { getStockLevel } from './stockLevel' import { getDevices } from './devices' +import { getBattery } from './battery' export class ExtractSelectedVariationError extends Data.TaggedError( 'ExtractSelectedVariationError', @@ -49,6 +50,7 @@ export type PageData = { color: string sim: string stockLevel: string + battery: string devices: DeviceData[] } @@ -76,8 +78,11 @@ export const getPageData = (page: Page) => const sim = yield* getSim(page) const stockLevel = yield* getStockLevel(page) + const devices = yield* getDevices(page) + const battery = yield* getBattery(page) + const pageData: PageData = { price, productName, @@ -87,6 +92,7 @@ export const getPageData = (page: Page) => sim, stockLevel, devices, + battery, } return pageData diff --git a/src/lib/server/crawler/clevertronik/index.ts b/src/lib/server/crawler/clevertronik/index.ts index 63afb17..67a72d9 100644 --- a/src/lib/server/crawler/clevertronik/index.ts +++ b/src/lib/server/crawler/clevertronik/index.ts @@ -1,14 +1,15 @@ -import { Data, Effect, pipe } from 'effect' +import { Data, Effect } from 'effect' import { openPage } from '../openPage' import * as Crawler from '../index' import { withOperation } from '$lib/server/utils' import { getProductName } from './data/productName' -import { getAllVariations, VariationEnum, type Variation } from './variations' -import { getPageData, type PageData } from './data' -import type { Page } from 'puppeteer' +import { + getAllVariations, + getVariationCategory, + VariationEnum, +} from './variations' import { getAllVariationCategoryData, - getAllVariationData, type VariationCategory, } from './variationData' @@ -30,37 +31,9 @@ export const crawlClevertronik = Effect.gen(function* () { yield* Effect.logInfo('opened clevertronik page') - let productName = yield* getProductName(page) + const variationCategory = yield* getVariationCategory(page) - const variations = yield* getAllVariations(page) - - // const variationCategory: VariationCategory = { - // name: 'Capacity', - // variations: variations.capacities, - // subVariationCategory: { - // name: 'Color', - // variations: variations.colors, - // subVariationCategory: { - // name: 'Sim', - // variations: variations.sims, - // subVariationCategory: { - // name: 'Condition', - // variations: variations.conditions, - // subVariationCategory: { - // name: 'Battery', - // variations: variations.battery, - // }, - // }, - // }, - // }, - // } - - const variationCategory: VariationCategory = { - name: VariationEnum.CAPACITY, - subVariationCategory: { - name: VariationEnum.COLOR, - }, - } + yield* Effect.logInfo(`Got tree`, { variationCategory }) const variationData = yield* getAllVariationCategoryData( page, @@ -70,9 +43,11 @@ export const crawlClevertronik = Effect.gen(function* () { yield* Effect.promise(() => page.browser().close()) return { - productName, - ...variations, + variationCategory, variationData, + // productName, + // ...variations, + // variationData, } }).pipe( Effect.provide(Crawler.CrawlerLayer()), diff --git a/src/lib/server/crawler/clevertronik/variationData/index.ts b/src/lib/server/crawler/clevertronik/variationData/index.ts index 12fd351..b4eb052 100644 --- a/src/lib/server/crawler/clevertronik/variationData/index.ts +++ b/src/lib/server/crawler/clevertronik/variationData/index.ts @@ -34,7 +34,7 @@ export const getAllVariationCategoryData = ( const variations = yield* getVariationHandler[variationCategory.name](variationPage) - for (const variation of variations.filter((v) => v.active)) { + for (const variation of variations?.filter((v) => v.active) ?? []) { if (!variation.selected) { yield* gotoVariation(variationPage, variation) } diff --git a/src/lib/server/crawler/clevertronik/variations/battery.ts b/src/lib/server/crawler/clevertronik/variations/battery.ts index 6c56c5b..d0f4e3c 100644 --- a/src/lib/server/crawler/clevertronik/variations/battery.ts +++ b/src/lib/server/crawler/clevertronik/variations/battery.ts @@ -6,5 +6,7 @@ const BATTERY_CONTAINER_SELECTOR = '.new_battery_options' export const getBatteryVariations = (page: Page) => Effect.gen(function* () { - return yield* getVariations(page, BATTERY_CONTAINER_SELECTOR, 'DIV') + return yield* getVariations(page, BATTERY_CONTAINER_SELECTOR, 'DIV').pipe( + Effect.catchTag('VariationParseError', () => Effect.succeed(undefined)), + ) }) diff --git a/src/lib/server/crawler/clevertronik/variations/index.ts b/src/lib/server/crawler/clevertronik/variations/index.ts index 5688055..a6fc7e7 100644 --- a/src/lib/server/crawler/clevertronik/variations/index.ts +++ b/src/lib/server/crawler/clevertronik/variations/index.ts @@ -6,6 +6,7 @@ import { getColorVariations } from './color' import { getSimVariations } from './sim' import { getConditionVariations } from './condition' import { getBatteryVariations } from './battery' +import type { VariationCategory } from '../variationData' export type Variation = { label: string @@ -27,12 +28,19 @@ export class VariationParseError extends Data.TaggedError( message?: string }> {} +export class VariationCategoryError extends Data.TaggedError( + 'VariationCategoryError', +)<{ + cause?: unknown + message?: string +}> {} + export enum VariationEnum { CAPACITY = 'Capacity', COLOR = 'Color', SIM = 'Sim', CONDITION = 'CONDITION', - BATTERY = 'BATTER', + BATTERY = 'BATTERY', } export const getVariationHandler = { @@ -116,3 +124,96 @@ export const getAllVariations = (page: Page) => battery, } }) + +const attributeVariationMapping: Record = { + prop_19: VariationEnum.CAPACITY, + prop_17: VariationEnum.COLOR, + prop_39: VariationEnum.SIM, + condition: VariationEnum.CONDITION, +} + +export const getVariationCategory = (page: Page) => + Effect.gen(function* () { + const attributes: string[] = yield* Effect.tryPromise({ + try: () => + page.evaluate(() => { + const divs = document.querySelectorAll('[data-scrollel]') + + const attributes: string[] = [] + + divs.forEach((div) => { + const val = div.getAttribute('data-scrollel') + if (!val) return + attributes.push(val) + }) + + return attributes + }), + catch: (cause) => + new VariationParseError({ + message: 'Could not get variation attributes', + cause, + }), + }) + + const hasBatteryVariation = yield* Effect.tryPromise({ + try: () => + page.evaluate(() => { + const batteryDiv: HTMLDivElement | null = document.querySelector( + '.new_battery_options', + ) + + return Boolean(batteryDiv) + }), + catch: (cause) => + new VariationParseError({ + message: 'Error getting possible battery variation', + cause, + }), + }) + + let variationCategory: VariationCategory | undefined + let currentCategory: VariationCategory | undefined + + for (const attr of attributes) { + const variationKey = attributeVariationMapping[attr] + if (!variationKey) continue + + if (!variationCategory) { + variationCategory = { + name: variationKey, + } + + currentCategory = variationCategory + continue + } + + if (!currentCategory) { + continue + } + + const newCategory: VariationCategory = { + name: variationKey, + } + + currentCategory.subVariationCategory = newCategory + + currentCategory = newCategory + } + + if (currentCategory && hasBatteryVariation) { + currentCategory.subVariationCategory = { + name: VariationEnum.BATTERY, + } + } + + if (!variationCategory) { + return yield* Effect.fail( + new VariationCategoryError({ + message: 'Unable to get variation category tree', + }), + ) + } + + return variationCategory + })