clock.c

Go to the documentation of this file.

 
#include <core/system.h>
#include <core/arch.h>
#include <core/kargs.h>
#include <core/string.h>
#include <platform/misc.h>
 
#define CALIBRATE_US    50000       /* 50 ms */
 
enum { SOURCE_PIT, SOURCE_HPET, SOURCE_TSC };
 
static const char *names[] = {"pit", "hpet", "tsc"};
 
static int source = SOURCE_PIT;
static uint64_t mult;               /* the ns per count, << 32 */
static uint64_t base_count, base_ns, last_ns;
 
/* the timer's ticks: their period, and how many there were */
static uint32_t tick_period;
static uint64_t ticks;

static inline uint64_t rdtsc(void)
{
    uint32_t lo, hi;
    asm volatile("rdtsc" : "=a"(lo), "=d"(hi));
    return (uint64_t) hi << 32 | lo;
}

static inline void cpuid(uint32_t leaf, uint32_t *a, uint32_t *b, uint32_t *c, uint32_t *d)
{
    asm volatile("cpuid" : "=a"(*a), "=b"(*b), "=c"(*c), "=d"(*d) : "a"(leaf), "c"(0));
}

static int tsc_invariant(void)
{
    uint32_t a, b, c, d;
 
    cpuid(0x80000000, &a, &b, &c, &d);
 
    if (a < 0x80000007)
        return 0;
 
    cpuid(0x80000007, &a, &b, &c, &d);
    return !!(d & (1 << 8));
}

static inline uint64_t mul_shift32(uint64_t a, uint64_t b)
{
    uint64_t ah = a >> 32, al = a & 0xffffffff;
    uint64_t bh = b >> 32, bl = b & 0xffffffff;
 
    return ((ah * bh) << 32) + ah * bl + al * bh + ((al * bl) >> 32);
}

static uint64_t counter(void)
{
    return source == SOURCE_TSC ? rdtsc() : x86_hpet_read32();
}

static uint64_t since(uint64_t c)
{
    return source == SOURCE_TSC ? c - base_count : (uint32_t) (c - base_count);
}
 
/*
 * Reads the clock.
 */
uint64_t arch_rtime_ns(void)
{
    if (source == SOURCE_PIT)
        return ticks * tick_period;
 
    uint64_t ns = base_ns + mul_shift32(since(counter()), mult);
 
    if (ns < last_ns)
        ns = last_ns;
 
    last_ns = ns;
    return ns;
}
 
/*
 * Reads the clock, in microseconds.
 */
uint64_t arch_rtime_us(void)
{
    return arch_rtime_ns() / 1000ULL;
}
 
/*
 * Reads the clock, in milliseconds.
 */
uint64_t arch_rtime_ms(void)
{
    return arch_rtime_ns() / 1000000ULL;
}

void x86_clock_tick(void)
{
    ticks++;
 
    if (source == SOURCE_PIT)
        return;
 
    uint64_t c = counter();
 
    base_ns += mul_shift32(since(c), mult);
    base_count = c;
}

static uint64_t tsc_calibrate(int hpet, const char **by)
{
    uint64_t t0, t1, ns;
 
    if (hpet) {
        uint64_t period = x86_hpet_period_fs();
        uint64_t h0 = x86_hpet_read(), h1;
        uint64_t want = CALIBRATE_US * 1000000000ULL / period;
 
        t0 = rdtsc();
        while ((h1 = x86_hpet_read()) - h0 < want)
            ;
        t1 = rdtsc();
 
        ns = (h1 - h0) * period / 1000000ULL;
        *by = "hpet";
    } else {
        t0 = rdtsc();
        x86_pit_delay(CALIBRATE_US);
        t1 = rdtsc();
 
        ns = CALIBRATE_US * 1000ULL;
        *by = "pit";
    }
 
    return t1 > t0 ? (ns << 32) / (t1 - t0) : 0;
}

void x86_clock_ticks(uint32_t period_ns)
{
    tick_period = period_ns;
 
    if (source == SOURCE_PIT)
        printk("clock: timer ticks, %d ns\n", period_ns);
}

int x86_clock_precise(void)
{
    return source != SOURCE_PIT;
}

void x86_clock_init(void)
{
    const char *want = NULL;
    int hpet = x86_hpet_init && !x86_hpet_init();
 
    kargs_get("clock", &want);
 
    int forced = want ? !strcmp(want, "tsc") ? SOURCE_TSC : !strcmp(want, "hpet") ? SOURCE_HPET :
                 !strcmp(want, "pit") ? SOURCE_PIT : -1 : -1;
 
    if (want && forced < 0)
        printk("clock: clock=%s: not tsc, hpet or pit\n", want);
 
    int choice = forced >= 0 ? forced : tsc_invariant() ? SOURCE_TSC : hpet ? SOURCE_HPET : SOURCE_PIT;
 
    if (choice == SOURCE_HPET && !hpet) {
        printk("clock: no HPET\n");
        choice = tsc_invariant() ? SOURCE_TSC : SOURCE_PIT;
    }
 
    if (choice == SOURCE_TSC && forced != SOURCE_TSC && !tsc_invariant())
        choice = SOURCE_PIT;
 
    if (choice == SOURCE_TSC) {
        const char *by;
 
        if (forced == SOURCE_TSC && !tsc_invariant())
            printk("clock: the TSC isn't invariant (its rate may change)\n");
 
        mult = tsc_calibrate(hpet, &by);
 
        if (mult) {
            uint64_t mhz = (1000ULL << 32) / mult;
            printk("clock: tsc, %ld MHz (calibrated against the %s)\n", mhz, by);
        } else {
            choice = hpet ? SOURCE_HPET : SOURCE_PIT;
        }
    }
 
    if (choice == SOURCE_HPET) {
        mult = (x86_hpet_period_fs() << 32) / 1000000ULL;
        printk("clock: hpet, %ld fs a count\n", x86_hpet_period_fs());
    }
 
    source = choice;
 
    if (source != SOURCE_PIT) {
        base_count = counter();
        base_ns = last_ns = 0;
    }
}

const char *x86_clock_source(void)
{
    return names[source];
}