airflow-repo/dags/sql/session_features.hql

WITH base AS (
    SELECT
        person_id,
        timestamp,
        event,
        properties,
        row_number() OVER (
            PARTITION BY person_id
            ORDER BY timestamp, event
        ) AS global_order
    FROM events
    WHERE timestamp >= toDateTime('{start_date}')
      AND timestamp < toDateTime('{end_date}')
      AND event NOT IN ('flutter_error', 'platform_error')
),

ordered_events AS (
    SELECT
        *,
        lagInFrame(timestamp) OVER (
            PARTITION BY person_id
            ORDER BY global_order
        ) AS prev_ts,

        lagInFrame(event) OVER (
            PARTITION BY person_id
            ORDER BY global_order
        ) AS prev_event
    FROM base
),

session_flags AS (
    SELECT
        *,
        CASE
            WHEN prev_ts IS NULL THEN 1
            WHEN event = 'Application Opened' THEN 1
            WHEN prev_event IN ('Application Backgrounded', 'Application Closed') THEN 1
            WHEN dateDiff('minute', prev_ts, timestamp) > 10 THEN 1
            ELSE 0
        END AS is_new_session
    FROM ordered_events
),

sessionized AS (
    SELECT
        *,
        sum(is_new_session) OVER (
            PARTITION BY person_id
            ORDER BY global_order
        ) AS session_number
    FROM session_flags
),

session_duration_calc AS (
    SELECT
        *,
        dateDiff(
            'second',
            MIN(timestamp) OVER (
                PARTITION BY person_id, session_number
            ),
            timestamp
        ) AS seconds_since_session_start
    FROM sessionized
),

split_sessions AS (
    SELECT
        *,
        floor(seconds_since_session_start / 1800) AS session_sub_id
    FROM session_duration_calc
),

session_events AS (
    SELECT
        *,
        row_number() OVER (
            PARTITION BY person_id, session_number, session_sub_id
            ORDER BY timestamp, event
        ) AS session_index
    FROM split_sessions
),

inter_event_calc AS (
    SELECT
        person_id,
        session_number,
        session_sub_id,
        timestamp,
        event,
        properties,
        session_index,

        lagInFrame(timestamp) OVER (
            PARTITION BY person_id, session_number, session_sub_id
            ORDER BY session_index
        ) AS prev_ts_in_session,

        CASE
            WHEN session_index = 1 THEN NULL
            ELSE greatest(
                0,
                dateDiff(
                    'second',
                    lagInFrame(timestamp) OVER (
                        PARTITION BY person_id, session_number, session_sub_id
                        ORDER BY session_index
                    ),
                    timestamp
                )
            )
        END AS inter_event_time_seconds
    FROM session_events
),

session_stats AS (
    SELECT
        person_id,
        session_number,
        session_sub_id,

        COUNT(*) AS event_count,

        uniqExact(event) AS unique_event_types,

        uniqExact(
            if(
                event = '$screen',
                replaceRegexpOne(
                    JSONExtractString(properties, '$screen_name'),
                    '\\?.*$',
                    ''
                ),
                NULL
            )
        ) AS screens_visited,

        MIN(timestamp) AS session_start,
        MAX(timestamp) AS session_end,

        dateDiff(
            'second',
            MIN(timestamp),
            MAX(timestamp)
        ) AS duration_seconds,

        COUNT(*) * 60.0 /
        greatest(
            dateDiff(
                'second',
                MIN(timestamp),
                MAX(timestamp)
            ),
            1
        ) AS events_per_minute,

        AVG(inter_event_time_seconds) AS inter_event_time_mean_seconds,

        median(inter_event_time_seconds) AS inter_event_time_median_seconds,

        max(inter_event_time_seconds) AS max_inter_event_gap_seconds,

        sqrt(
            varSamp(inter_event_time_seconds)
        ) AS inter_event_time_std_seconds,

        toHour(MIN(timestamp)) AS session_start_hour,

        toDayOfWeek(MIN(timestamp)) AS session_day_of_week,

        uniqExact(
            toStartOfHour(timestamp)
        ) AS distinct_hours_active

    FROM inter_event_calc
    GROUP BY
        person_id,
        session_number,
        session_sub_id

    HAVING
        COUNT(*) > 2
        AND dateDiff(
            'second',
            MIN(timestamp),
            MAX(timestamp)
        ) >= 4
)

SELECT
    person_id,

    concat(
        toString(person_id),
        '_',
        toString(session_number),
        '_',
        toString(session_sub_id)
    ) AS session_id,

    session_start,
    -- 24-Hour Cyclical Encoding for Session Start
    sin(2 * pi() * (toUnixTimestamp(session_start) - toUnixTimestamp(toStartOfDay(session_start))) / 86400) AS session_start_sin,
    cos(2 * pi() * (toUnixTimestamp(session_start) - toUnixTimestamp(toStartOfDay(session_start))) / 86400) AS session_start_cos,

    session_end,
    -- 24-Hour Cyclical Encoding for Session End
    sin(2 * pi() * (toUnixTimestamp(session_end) - toUnixTimestamp(toStartOfDay(session_end))) / 86400) AS session_end_sin,
    cos(2 * pi() * (toUnixTimestamp(session_end) - toUnixTimestamp(toStartOfDay(session_end))) / 86400) AS session_end_cos,

    event_count,
    unique_event_types,
    screens_visited,

    duration_seconds,
    events_per_minute,

    inter_event_time_mean_seconds,
    inter_event_time_median_seconds,
    inter_event_time_std_seconds,
    max_inter_event_gap_seconds,

    session_start_hour,
    
    session_day_of_week,
    -- 7-Day Cyclical Encoding for Day of the Week
    sin(2 * pi() * session_day_of_week / 7) AS session_day_of_week_sin,
    cos(2 * pi() * session_day_of_week / 7) AS session_day_of_week_cos,
    
    distinct_hours_active

FROM session_stats
ORDER BY session_start DESC