[PATCH v2 6/9] ram: octeon: Add MIPS Octeon3 DDR4 support (part 2/3)

From: Aaron Williams awilliams@marvell.com

This Octeon 3 DDR driver is ported from the 2013 Cavium / Marvell U-Boot repository. It currently supports DDR4 on Octeon 3. It can be later extended to support also DDR3 and Octeon 2 platforms.

Part 2 includes the very complex Octeon 3 DDR4 configuration

Signed-off-by: Aaron Williams awilliams@marvell.com Signed-off-by: Stefan Roese sr@denx.de ---

(no changes since v1)

drivers/ram/octeon/octeon3_lmc.c | 11484 +++++++++++++++++++++++++++++ 1 file changed, 11484 insertions(+) create mode 100644 drivers/ram/octeon/octeon3_lmc.c

diff --git a/drivers/ram/octeon/octeon3_lmc.c b/drivers/ram/octeon/octeon3_lmc.c new file mode 100644 index 0000000000..21e331897b --- /dev/null +++ b/drivers/ram/octeon/octeon3_lmc.c @@ -0,0 +1,11484 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Marvell International Ltd. + * + * https://spdx.org/licenses + */ + +#include <command.h> +#include <dm.h> +#include <hang.h> +#include <i2c.h> +#include <ram.h> +#include <time.h> + +#include <linux/bitops.h> +#include <linux/io.h> + +#include <mach/octeon_ddr.h> + +/* Random number generator stuff */ + +#define CVMX_RNM_CTL_STATUS 0x0001180040000000 +#define CVMX_OCT_DID_RNG 8ULL + +static u64 cvmx_build_io_address(u64 major_did, u64 sub_did) +{ + return ((0x1ull << 48) | (major_did << 43) | (sub_did << 40)); +} + +static u64 cvmx_rng_get_random64(void) +{ + return csr_rd(cvmx_build_io_address(CVMX_OCT_DID_RNG, 0)); +} + +static void cvmx_rng_enable(void) +{ + u64 val; + + val = csr_rd(CVMX_RNM_CTL_STATUS); + val |= BIT(0) | BIT(1); + csr_wr(CVMX_RNM_CTL_STATUS, val); +} + +#define CAVIUM_ONLY + +// FIXME: set to 1 for development-level messages, 0 for production +#define RLEVEL_PRINTALL_DEFAULT 1 +#define WLEVEL_PRINTALL_DEFAULT 1 + +#define ALLOW_BY_RANK_INIT 1 +#define USE_ORIG_TEST_DRAM_BYTE 0 + +#define RLEXTRAS_PATCH 1 // write to unused RL rank entries +#define WLEXTRAS_PATCH 1 // write to unused WL rank entries +#define ADD_48_OHM_SKIP 0 +#define NOSKIP_40_48_OHM 1 +#define NOSKIP_48_STACKED 1 +#define NOSKIP_FOR_MINI 1 +#define NOSKIP_FOR_2S_1R 1 + +// define how many HW RL samples per rank to take +// multiple samples will allow looking for the best sample score +// +#define RLEVEL_SAMPLES_DEFAULT 3 + +#define FAILSAFE_CHECK 1 + +#define PERFECT_BITMASK_COUNTING 1 +#define PRINT_PERFECT_COUNTS PERFECT_BITMASK_COUNTING && 1 + +#define DISABLE_SW_WL_PASS_2 1 +#define SW_WL_CHECK_PATCH 1 +#define HW_WL_MAJORITY 1 +// try HW WL base alternate if available when SW WL fails +#define SWL_TRY_HWL_ALT HW_WL_MAJORITY && 1 +// define how many HW WL samples to take for majority voting +// MUST BE odd!! +// assume there should only be 2 possible values that will show up, +// so treat ties as a problem!!! +// NOTE: do not change this without checking the code!!! +#define WLEVEL_LOOPS_DEFAULT 5 + +#define SWL_WITH_HW_ALTS_CHOOSE_SW 0 // FIXME: allow override? + +#define SW_WLEVEL_HW_DEFAULT 1 +// collect and print LMC utilization using SWL software algorithm +#define ENABLE_SW_WLEVEL_UTILIZATION 1 + +#define COUNT_RL_CANDIDATES 1 + +#define DDR3_DAC_OVERRIDE 1 + +#define DEFAULT_BEST_RANK_SCORE 9999999 +#define MAX_RANK_SCORE_LIMIT 99 // FIXME? + +#define ENABLE_COMPUTED_VREF_ADJUSTMENT 1 + +#undef ENABLE_SLOT_CTL_ACCESS + +#undef DEBUG_PERFORM_DDR3_SEQUENCE +#define ddr_seq_print(format, ...) do {} while (0) + +struct wlevel_bitcnt { + int bitcnt[4]; +}; + +static void display_dac_dbi_settings(int lmc, int dac_or_dbi, + int ecc_ena, int *settings, char *title); + +static unsigned short load_dac_override(struct ddr_priv *priv, int if_num, + int dac_value, int byte); + +// "mode" arg +#define DBTRAIN_TEST 0 +#define DBTRAIN_DBI 1 +#define DBTRAIN_LFSR 2 + +static int run_best_hw_patterns(struct ddr_priv *priv, int lmc, u64 phys_addr, + int mode, u64 *xor_data); + +#define LMC_DDR3_RESET_ASSERT 0 +#define LMC_DDR3_RESET_DEASSERT 1 + +static void cn7xxx_lmc_ddr3_reset(struct ddr_priv *priv, int if_num, int reset) +{ + union cvmx_lmcx_reset_ctl reset_ctl; + + /* + * 4. Deassert DDRn_RESET_L pin by writing + * LMC(0..3)_RESET_CTL[DDR3RST] = 1 + * without modifying any other LMC(0..3)_RESET_CTL fields. + * 5. Read LMC(0..3)_RESET_CTL and wait for the result. + * 6. Wait a minimum of 500us. This guarantees the necessary T = 500us + * delay between DDRn_RESET_L deassertion and DDRn_DIMM*_CKE* + * assertion. + */ + debug("LMC%d %s DDR_RESET_L\n", if_num, + (reset == + LMC_DDR3_RESET_DEASSERT) ? "De-asserting" : "Asserting"); + + reset_ctl.u64 = lmc_rd(priv, CVMX_LMCX_RESET_CTL(if_num)); + reset_ctl.cn78xx.ddr3rst = reset; + lmc_wr(priv, CVMX_LMCX_RESET_CTL(if_num), reset_ctl.u64); + + lmc_rd(priv, CVMX_LMCX_RESET_CTL(if_num)); + + udelay(500); +} + +static void perform_lmc_reset(struct ddr_priv *priv, int node, int if_num) +{ + /* + * 5.9.6 LMC RESET Initialization + * + * The purpose of this step is to assert/deassert the RESET# pin at the + * DDR3/DDR4 parts. + * + * This LMC RESET step is done for all enabled LMCs. + * + * It may be appropriate to skip this step if the DDR3/DDR4 DRAM parts + * are in self refresh and are currently preserving their + * contents. (Software can determine this via + * LMC(0..3)_RESET_CTL[DDR3PSV] in some circumstances.) The remainder of + * this section assumes that the DRAM contents need not be preserved. + * + * The remainder of this section assumes that the CN78XX DDRn_RESET_L + * pin is attached to the RESET# pin of the attached DDR3/DDR4 parts, + * as will be appropriate in many systems. + * + * (In other systems, such as ones that can preserve DDR3/DDR4 part + * contents while CN78XX is powered down, it will not be appropriate to + * directly attach the CN78XX DDRn_RESET_L pin to DRESET# of the + * DDR3/DDR4 parts, and this section may not apply.) + * + * The remainder of this section describes the sequence for LMCn. + * + * Perform the following six substeps for LMC reset initialization: + * + * 1. If not done already, assert DDRn_RESET_L pin by writing + * LMC(0..3)_RESET_ CTL[DDR3RST] = 0 without modifying any other + * LMC(0..3)_RESET_CTL fields. + */ + + if (!ddr_memory_preserved(priv)) { + /* + * 2. Read LMC(0..3)_RESET_CTL and wait for the result. + */ + + lmc_rd(priv, CVMX_LMCX_RESET_CTL(if_num)); + + /* + * 3. Wait until RESET# assertion-time requirement from JEDEC + * DDR3/DDR4 specification is satisfied (200 us during a + * power-on ramp, 100ns when power is already stable). + */ + + udelay(200); + + /* + * 4. Deassert DDRn_RESET_L pin by writing + * LMC(0..3)_RESET_CTL[DDR3RST] = 1 + * without modifying any other LMC(0..3)_RESET_CTL fields. + * 5. Read LMC(0..3)_RESET_CTL and wait for the result. + * 6. Wait a minimum of 500us. This guarantees the necessary + * T = 500us delay between DDRn_RESET_L deassertion and + * DDRn_DIMM*_CKE* assertion. + */ + cn7xxx_lmc_ddr3_reset(priv, if_num, LMC_DDR3_RESET_DEASSERT); + + /* Toggle Reset Again */ + /* That is, assert, then de-assert, one more time */ + cn7xxx_lmc_ddr3_reset(priv, if_num, LMC_DDR3_RESET_ASSERT); + cn7xxx_lmc_ddr3_reset(priv, if_num, LMC_DDR3_RESET_DEASSERT); + } +} + +void oct3_ddr3_seq(struct ddr_priv *priv, int rank_mask, int if_num, + int sequence) +{ + /* + * 3. Without changing any other fields in LMC(0)_CONFIG, write + * LMC(0)_CONFIG[RANKMASK] then write both + * LMC(0)_SEQ_CTL[SEQ_SEL,INIT_START] = 1 with a single CSR write + * operation. LMC(0)_CONFIG[RANKMASK] bits should be set to indicate + * the ranks that will participate in the sequence. + * + * The LMC(0)_SEQ_CTL[SEQ_SEL] value should select power-up/init or + * selfrefresh exit, depending on whether the DRAM parts are in + * self-refresh and whether their contents should be preserved. While + * LMC performs these sequences, it will not perform any other DDR3 + * transactions. When the sequence is complete, hardware sets the + * LMC(0)_CONFIG[INIT_STATUS] bits for the ranks that have been + * initialized. + * + * If power-up/init is selected immediately following a DRESET + * assertion, LMC executes the sequence described in the "Reset and + * Initialization Procedure" section of the JEDEC DDR3 + * specification. This includes activating CKE, writing all four DDR3 + * mode registers on all selected ranks, and issuing the required + * ZQCL + * command. The LMC(0)_CONFIG[RANKMASK] value should select all ranks + * with attached DRAM in this case. If LMC(0)_CONTROL[RDIMM_ENA] = 1, + * LMC writes the JEDEC standard SSTE32882 control words selected by + * LMC(0)_DIMM_CTL[DIMM*_WMASK] between DDR_CKE* signal assertion and + * the first DDR3 mode register write operation. + * LMC(0)_DIMM_CTL[DIMM*_WMASK] should be cleared to 0 if the + * corresponding DIMM is not present. + * + * If self-refresh exit is selected, LMC executes the required SRX + * command followed by a refresh and ZQ calibration. Section 4.5 + * describes behavior of a REF + ZQCS. LMC does not write the DDR3 + * mode registers as part of this sequence, and the mode register + * parameters must match at self-refresh entry and exit times. + * + * 4. Read LMC(0)_SEQ_CTL and wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] + * to be set. + * + * 5. Read LMC(0)_CONFIG[INIT_STATUS] and confirm that all ranks have + * been initialized. + */ + + union cvmx_lmcx_seq_ctl seq_ctl; + union cvmx_lmcx_config lmc_config; + int timeout; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + lmc_config.s.rankmask = rank_mask; + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64); + + seq_ctl.u64 = 0; + + seq_ctl.s.init_start = 1; + seq_ctl.s.seq_sel = sequence; + + ddr_seq_print + ("Performing LMC sequence: rank_mask=0x%02x, sequence=0x%x, %s\n", + rank_mask, sequence, sequence_str[sequence]); + + if (seq_ctl.s.seq_sel == 3) + debug("LMC%d: Exiting Self-refresh Rank_mask:%x\n", if_num, + rank_mask); + + lmc_wr(priv, CVMX_LMCX_SEQ_CTL(if_num), seq_ctl.u64); + lmc_rd(priv, CVMX_LMCX_SEQ_CTL(if_num)); + + timeout = 100; + do { + udelay(100); /* Wait a while */ + seq_ctl.u64 = lmc_rd(priv, CVMX_LMCX_SEQ_CTL(if_num)); + if (--timeout == 0) { + printf("Sequence %d timed out\n", sequence); + break; + } + } while (seq_ctl.s.seq_complete != 1); + + ddr_seq_print(" LMC sequence=%x: Completed.\n", sequence); +} + +#if USE_ORIG_TEST_DRAM_BYTE + +#define DO_LIKE_RANDOM_XOR 1 + +static int test_dram_byte64(struct ddr_priv *priv, int if_num, u64 p, + u64 bitmask, u64 *xor_data) +{ + u64 p1, p2, d1, d2; + u64 v, v1; + u64 p2offset = 0x4000000; + u64 datamask; + u64 xor; + int i, j, k; + int errors = 0; + int index; + int bitno = (octeon_is_cpuid(OCTEON_CN7XXX)) ? 20 : 18; +#if DO_LIKE_RANDOM_XOR + u64 pattern1 = cvmx_rng_get_random64(); + u64 this_pattern; +#endif + u64 bad_bits[2] = { 0, 0 }; + int node = 0; + + if (octeon_is_cpuid(OCTEON_CN73XX) || octeon_is_cpuid(OCTEON_CNF75XX)) + bitno = 18; + + // When doing in parallel, the caller must provide full 8-byte bitmask. + // Byte lanes may be clear in the mask to indicate no testing on that + // lane. + datamask = bitmask; + + // final address must include LMC and node + p |= (if_num << 7); /* Map address into proper interface */ + p |= (u64)node << CVMX_NODE_MEM_SHIFT; // map to node + p |= 1ull << 63; + + /* Add offset to both test regions to not clobber boot stuff + * when running from L2. + */ + p += 0x4000000; + + debug("N%d.LMC%d: %s: phys_addr=0x%lx/0x%lx (0x%lx)\n", + node, if_num, __func__, p, p + p2offset, 1ULL << bitno); + + /* + * The loop ranges and increments walk through a range of addresses + * avoiding bits that alias to different memory interfaces (LMCs) + * on the CN88XX; ie we want to limit activity to a single memory + * channel. + */ + + /* Store something into each location first */ + // NOTE: the ordering of loops is purposeful + for (i = 0; i < (1 << 7); i += 8) { + for (k = 0; k < (1 << bitno); k += (1 << 14)) { + for (j = 0; j < (1 << 12); j += (1 << 9)) { + index = i + j + k; + p1 = p + index; + p2 = p1 + p2offset; + +#if DO_LIKE_RANDOM_XOR + v = pattern1 * p1; + v1 = v; // write the same thing to both areas +#else + v = 0ULL; + v1 = v; +#endif + cvmx_write64_uint64(p1, v); + cvmx_write64_uint64(p2, v1); + + /* Write back and invalidate the cache lines + * + * For OCX we cannot limit the number of L2 ways + * so instead we just write back and invalidate + * the L2 cache lines. This is not possible + * when booting remotely, however so this is + * only enabled for U-Boot right now. + * Potentially the BDK can also take advantage + * of this. + */ + CVMX_CACHE_WBIL2(p1, 0); + CVMX_CACHE_WBIL2(p2, 0); + } + } + } + CVMX_DCACHE_INVALIDATE; + +#if DO_LIKE_RANDOM_XOR + this_pattern = cvmx_rng_get_random64(); +#endif + + // modify the contents of each location in some way + // NOTE: the ordering of loops is purposeful + for (i = 0; i < (1 << 7); i += 8) { + for (k = 0; k < (1 << bitno); k += (1 << 14)) { + for (j = 0; j < (1 << 12); j += (1 << 9)) { + index = i + j + k; + p1 = p + index; + p2 = p1 + p2offset; +#if DO_LIKE_RANDOM_XOR + v = cvmx_read64_uint64(p1) ^ this_pattern; + v1 = cvmx_read64_uint64(p2) ^ this_pattern; +#else + v = test_pattern[index % + (sizeof(test_pattern) / + sizeof(u64))]; + v &= datamask; + v1 = ~v; +#endif + + cvmx_write64_uint64(p1, v); + cvmx_write64_uint64(p2, v1); + + /* + * Write back and invalidate the cache lines + * + * For OCX we cannot limit the number of L2 ways + * so instead we just write back and invalidate + * the L2 cache lines. This is not possible + * when booting remotely, however so this is + * only enabled for U-Boot right now. + * Potentially the BDK can also take advantage + * of this. + */ + CVMX_CACHE_WBIL2(p1, 0); + CVMX_CACHE_WBIL2(p2, 0); + } + } + } + CVMX_DCACHE_INVALIDATE; + + // test the contents of each location by predicting what should be there + // NOTE: the ordering of loops is purposeful: test full cachelines to + // detect an error occurring in any slot thereof + for (k = 0; k < (1 << bitno); k += (1 << 14)) { + for (j = 0; j < (1 << 12); j += (1 << 9)) { + for (i = 0; i < (1 << 7); i += 8) { + int bybit = 1; + u64 bymsk = 0xffULL; // start in byte lane 0 + + index = i + j + k; + p1 = p + index; + p2 = p1 + p2offset; +#if DO_LIKE_RANDOM_XOR + // FIXME: this should predict what we find...??? + v = (p1 * pattern1) ^ this_pattern; + d1 = cvmx_read64_uint64(p1); + d2 = cvmx_read64_uint64(p2); +#else + v = test_pattern[index % + (sizeof(test_pattern) / + sizeof(u64))]; + d1 = cvmx_read64_uint64(p1); + d2 = ~cvmx_read64_uint64(p2); +#endif + + // union of error bits only in active byte lanes + xor = ((d1 ^ v) | (d2 ^ v)) & datamask; + + if (!xor) + continue; + + // accumulate bad bits + bad_bits[0] |= xor; + + while (xor != 0) { + debug("ERROR: [0x%016llX] [0x%016llX] expected 0x%016llX xor %016llX\n", + p1, p2, v, xor); + // error(s) in this lane + if (xor & bymsk) { + // set the byte error bit + errors |= bybit; + // clear byte lane in error bits + xor &= ~bymsk; + // clear the byte lane in the + // mask + datamask &= ~bymsk; + // nothing left to do + if (datamask == 0) { + // completely done when + // errors found in all + // byte lanes in + // datamask + goto done_now; + } + } + // move mask into next byte lane + bymsk <<= 8; + // move bit into next byte position + bybit <<= 1; + } + } + } + } + +done_now: + if (xor_data) { // send the bad bits back... + xor_data[0] = bad_bits[0]; + xor_data[1] = bad_bits[1]; // let it be zeroed + } + return errors; +} + +#else /* USE_ORIG_TEST_DRAM_BYTE */ + +#define bdk_numa_get_address(n, p) ((p) | ((u64)n) << CVMX_NODE_MEM_SHIFT) +#define AREA_BASE_OFFSET BIT_ULL(26) + +static int test_dram_byte64(struct ddr_priv *priv, int lmc, u64 p, + u64 bitmask, u64 *xor_data) +{ + u64 p1, p2, d1, d2; + u64 v, v1; + u64 p2offset = (1ULL << 26); // offset to area 2 + u64 datamask; + u64 xor; + u64 i, j, k; + u64 ii; + int errors = 0; + //u64 index; + u64 pattern1 = cvmx_rng_get_random64(); + u64 pattern2 = 0; + u64 bad_bits[2] = { 0, 0 }; + int kbitno = (octeon_is_cpuid(OCTEON_CN7XXX)) ? 20 : 18; + union cvmx_l2c_ctl l2c_ctl; + int burst; + int saved_dissblkdty; + int node = 0; + + // Force full cacheline write-backs to boost traffic + l2c_ctl.u64 = l2c_rd(priv, CVMX_L2C_CTL); + saved_dissblkdty = l2c_ctl.cn78xx.dissblkdty; + l2c_ctl.cn78xx.dissblkdty = 1; + l2c_wr(priv, CVMX_L2C_CTL, l2c_ctl.u64); + + if (octeon_is_cpuid(OCTEON_CN73XX) || octeon_is_cpuid(OCTEON_CNF75XX)) + kbitno = 18; + + // Byte lanes may be clear in the mask to indicate no testing on that + //lane. + datamask = bitmask; + + /* + * Add offset to both test regions to not clobber boot stuff + * when running from L2 for NAND boot. + */ + p += AREA_BASE_OFFSET; // make sure base is out of the way of boot + + // final address must include LMC and node + p |= (lmc << 7); /* Map address into proper interface */ + p = bdk_numa_get_address(node, p); /* Map to node */ + p |= 1ull << 63; + +#define II_INC BIT_ULL(22) +#define II_MAX BIT_ULL(22) +#define K_INC BIT_ULL(14) +#define K_MAX BIT_ULL(kbitno) +#define J_INC BIT_ULL(9) +#define J_MAX BIT_ULL(12) +#define I_INC BIT_ULL(3) +#define I_MAX BIT_ULL(7) + + debug("N%d.LMC%d: %s: phys_addr=0x%llx/0x%llx (0x%llx)\n", + node, lmc, __func__, p, p + p2offset, 1ULL << kbitno); + + // loops are ordered so that only a single 64-bit slot is written to + // each cacheline at one time, then the cachelines are forced out; + // this should maximize read/write traffic + + // FIXME? extend the range of memory tested!! + for (ii = 0; ii < II_MAX; ii += II_INC) { + for (i = 0; i < I_MAX; i += I_INC) { + for (k = 0; k < K_MAX; k += K_INC) { + for (j = 0; j < J_MAX; j += J_INC) { + p1 = p + ii + k + j; + p2 = p1 + p2offset; + + v = pattern1 * (p1 + i); + // write the same thing to both areas + v1 = v; + + cvmx_write64_uint64(p1 + i, v); + cvmx_write64_uint64(p2 + i, v1); + + CVMX_CACHE_WBIL2(p1, 0); + CVMX_CACHE_WBIL2(p2, 0); + } + } + } + } + + CVMX_DCACHE_INVALIDATE; + + debug("N%d.LMC%d: dram_tuning_mem_xor: done INIT loop\n", node, lmc); + + /* Make a series of passes over the memory areas. */ + + for (burst = 0; burst < 1 /* was: dram_tune_use_bursts */ ; burst++) { + u64 this_pattern = cvmx_rng_get_random64(); + + pattern2 ^= this_pattern; + + /* + * XOR the data with a random value, applying the change to both + * memory areas. + */ + + // FIXME? extend the range of memory tested!! + for (ii = 0; ii < II_MAX; ii += II_INC) { + // FIXME: rearranged, did not make much difference? + for (i = 0; i < I_MAX; i += I_INC) { + for (k = 0; k < K_MAX; k += K_INC) { + for (j = 0; j < J_MAX; j += J_INC) { + p1 = p + ii + k + j; + p2 = p1 + p2offset; + + v = cvmx_read64_uint64(p1 + + i) ^ + this_pattern; + v1 = cvmx_read64_uint64(p2 + + i) ^ + this_pattern; + + cvmx_write64_uint64(p1 + i, v); + cvmx_write64_uint64(p2 + i, v1); + + CVMX_CACHE_WBIL2(p1, 0); + CVMX_CACHE_WBIL2(p2, 0); + } + } + } + } + + CVMX_DCACHE_INVALIDATE; + + debug("N%d.LMC%d: dram_tuning_mem_xor: done MODIFY loop\n", + node, lmc); + + /* + * Look for differences in the areas. If there is a mismatch, + * reset both memory locations with the same pattern. Failing + * to do so means that on all subsequent passes the pair of + * locations remain out of sync giving spurious errors. + */ + + // FIXME: Change the loop order so that an entire cache line + // is compared at one time. This is so that a read + // error that occurs *anywhere* on the cacheline will + // be caught, rather than comparing only 1 cacheline + // slot at a time, where an error on a different + // slot will be missed that time around + // Does the above make sense? + + // FIXME? extend the range of memory tested!! + for (ii = 0; ii < II_MAX; ii += II_INC) { + for (k = 0; k < K_MAX; k += K_INC) { + for (j = 0; j < J_MAX; j += J_INC) { + p1 = p + ii + k + j; + p2 = p1 + p2offset; + + // process entire cachelines in the + //innermost loop + for (i = 0; i < I_MAX; i += I_INC) { + int bybit = 1; + // start in byte lane 0 + u64 bymsk = 0xffULL; + + // FIXME: this should predict + // what we find...??? + v = ((p1 + i) * pattern1) ^ + pattern2; + d1 = cvmx_read64_uint64(p1 + i); + d2 = cvmx_read64_uint64(p2 + i); + + // union of error bits only in + // active byte lanes + xor = ((d1 ^ v) | (d2 ^ v)) & + datamask; + + if (!xor) + continue; + + // accumulate bad bits + bad_bits[0] |= xor; + + while (xor != 0) { + debug("ERROR(%03d): [0x%016llX] [0x%016llX] expected 0x%016llX d1 %016llX d2 %016llX\n", + burst, p1, p2, v, + d1, d2); + // error(s) in this lane + if (xor & bymsk) { + // set the byte + // error bit + errors |= bybit; + // clear byte + // lane in + // error bits + xor &= ~bymsk; + // clear the + // byte lane in + // the mask + datamask &= ~bymsk; +#if EXIT_WHEN_ALL_LANES_HAVE_ERRORS + // nothing + // left to do + if (datamask == 0) { + return errors; + } +#endif /* EXIT_WHEN_ALL_LANES_HAVE_ERRORS */ + } + // move mask into + // next byte lane + bymsk <<= 8; + // move bit into next + // byte position + bybit <<= 1; + } + } + CVMX_CACHE_WBIL2(p1, 0); + CVMX_CACHE_WBIL2(p2, 0); + } + } + } + + debug("N%d.LMC%d: dram_tuning_mem_xor: done TEST loop\n", + node, lmc); + } + + if (xor_data) { // send the bad bits back... + xor_data[0] = bad_bits[0]; + xor_data[1] = bad_bits[1]; // let it be zeroed + } + + // Restore original setting that could enable partial cacheline writes + l2c_ctl.u64 = l2c_rd(priv, CVMX_L2C_CTL); + l2c_ctl.cn78xx.dissblkdty = saved_dissblkdty; + l2c_wr(priv, CVMX_L2C_CTL, l2c_ctl.u64); + + return errors; +} +#endif /* USE_ORIG_TEST_DRAM_BYTE */ + +static void ddr4_mrw(struct ddr_priv *priv, int if_num, int rank, + int mr_wr_addr, int mr_wr_sel, int mr_wr_bg1) +{ + union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl; + + lmc_mr_mpr_ctl.u64 = 0; + lmc_mr_mpr_ctl.cn78xx.mr_wr_addr = (mr_wr_addr == -1) ? 0 : mr_wr_addr; + lmc_mr_mpr_ctl.cn78xx.mr_wr_sel = mr_wr_sel; + lmc_mr_mpr_ctl.cn78xx.mr_wr_rank = rank; + lmc_mr_mpr_ctl.cn78xx.mr_wr_use_default_value = + (mr_wr_addr == -1) ? 1 : 0; + lmc_mr_mpr_ctl.cn78xx.mr_wr_bg1 = mr_wr_bg1; + lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64); + + /* Mode Register Write */ + oct3_ddr3_seq(priv, 1 << rank, if_num, 0x8); +} + +#define INV_A0_17(x) ((x) ^ 0x22bf8) + +static void set_mpr_mode(struct ddr_priv *priv, int rank_mask, + int if_num, int dimm_count, int mpr, int bg1) +{ + int rankx; + + debug("All Ranks: Set mpr mode = %x %c-side\n", + mpr, (bg1 == 0) ? 'A' : 'B'); + + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + if (bg1 == 0) { + /* MR3 A-side */ + ddr4_mrw(priv, if_num, rankx, mpr << 2, 3, bg1); + } else { + /* MR3 B-side */ + ddr4_mrw(priv, if_num, rankx, INV_A0_17(mpr << 2), ~3, + bg1); + } + } +} + +static void do_ddr4_mpr_read(struct ddr_priv *priv, int if_num, + int rank, int page, int location) +{ + union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl; + + lmc_mr_mpr_ctl.u64 = lmc_rd(priv, CVMX_LMCX_MR_MPR_CTL(if_num)); + lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = 0; + lmc_mr_mpr_ctl.cn70xx.mr_wr_sel = page; /* Page */ + lmc_mr_mpr_ctl.cn70xx.mr_wr_rank = rank; + lmc_mr_mpr_ctl.cn70xx.mpr_loc = location; + lmc_mr_mpr_ctl.cn70xx.mpr_wr = 0; /* Read=0, Write=1 */ + lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64); + + /* MPR register access sequence */ + oct3_ddr3_seq(priv, 1 << rank, if_num, 0x9); + + debug("LMC_MR_MPR_CTL : 0x%016llx\n", + lmc_mr_mpr_ctl.u64); + debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_addr: 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mr_wr_addr); + debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_sel : 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mr_wr_sel); + debug("lmc_mr_mpr_ctl.cn70xx.mpr_loc : 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mpr_loc); + debug("lmc_mr_mpr_ctl.cn70xx.mpr_wr : 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mpr_wr); +} + +static int set_rdimm_mode(struct ddr_priv *priv, int if_num, int enable) +{ + union cvmx_lmcx_control lmc_control; + int save_rdimm_mode; + + lmc_control.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + save_rdimm_mode = lmc_control.s.rdimm_ena; + lmc_control.s.rdimm_ena = enable; + debug("Setting RDIMM_ENA = %x\n", enable); + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), lmc_control.u64); + + return save_rdimm_mode; +} + +static void ddr4_mpr_read(struct ddr_priv *priv, int if_num, int rank, + int page, int location, u64 *mpr_data) +{ + do_ddr4_mpr_read(priv, if_num, rank, page, location); + + mpr_data[0] = lmc_rd(priv, CVMX_LMCX_MPR_DATA0(if_num)); +} + +/* Display MPR values for Page */ +static void display_mpr_page(struct ddr_priv *priv, int rank_mask, + int if_num, int page) +{ + int rankx, location; + u64 mpr_data[3]; + + for (rankx = 0; rankx < 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + debug("N0.LMC%d.R%d: MPR Page %d loc [0:3]: ", + if_num, rankx, page); + for (location = 0; location < 4; location++) { + ddr4_mpr_read(priv, if_num, rankx, page, location, + mpr_data); + debug("0x%02llx ", mpr_data[0] & 0xFF); + } + debug("\n"); + + } /* for (rankx = 0; rankx < 4; rankx++) */ +} + +static void ddr4_mpr_write(struct ddr_priv *priv, int if_num, int rank, + int page, int location, u8 mpr_data) +{ + union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl; + + lmc_mr_mpr_ctl.u64 = 0; + lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = mpr_data; + lmc_mr_mpr_ctl.cn70xx.mr_wr_sel = page; /* Page */ + lmc_mr_mpr_ctl.cn70xx.mr_wr_rank = rank; + lmc_mr_mpr_ctl.cn70xx.mpr_loc = location; + lmc_mr_mpr_ctl.cn70xx.mpr_wr = 1; /* Read=0, Write=1 */ + lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64); + + /* MPR register access sequence */ + oct3_ddr3_seq(priv, 1 << rank, if_num, 0x9); + + debug("LMC_MR_MPR_CTL : 0x%016llx\n", + lmc_mr_mpr_ctl.u64); + debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_addr: 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mr_wr_addr); + debug("lmc_mr_mpr_ctl.cn70xx.mr_wr_sel : 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mr_wr_sel); + debug("lmc_mr_mpr_ctl.cn70xx.mpr_loc : 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mpr_loc); + debug("lmc_mr_mpr_ctl.cn70xx.mpr_wr : 0x%02x\n", + lmc_mr_mpr_ctl.cn70xx.mpr_wr); +} + +static void set_vref(struct ddr_priv *priv, int if_num, int rank, + int range, int value) +{ + union cvmx_lmcx_mr_mpr_ctl lmc_mr_mpr_ctl; + union cvmx_lmcx_modereg_params3 lmc_modereg_params3; + int mr_wr_addr = 0; + + lmc_mr_mpr_ctl.u64 = 0; + lmc_modereg_params3.u64 = lmc_rd(priv, + CVMX_LMCX_MODEREG_PARAMS3(if_num)); + + /* A12:A10 tCCD_L */ + mr_wr_addr |= lmc_modereg_params3.s.tccd_l << 10; + mr_wr_addr |= 1 << 7; /* A7 1 = Enable(Training Mode) */ + mr_wr_addr |= range << 6; /* A6 vrefDQ Training Range */ + mr_wr_addr |= value << 0; /* A5:A0 vrefDQ Training Value */ + + lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = mr_wr_addr; + lmc_mr_mpr_ctl.cn70xx.mr_wr_sel = 6; /* Write MR6 */ + lmc_mr_mpr_ctl.cn70xx.mr_wr_rank = rank; + lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64); + + /* 0x8 = Mode Register Write */ + oct3_ddr3_seq(priv, 1 << rank, if_num, 0x8); + + /* + * It is vendor specific whether vref_value is captured with A7=1. + * A subsequent MRS might be necessary. + */ + oct3_ddr3_seq(priv, 1 << rank, if_num, 0x8); + + mr_wr_addr &= ~(1 << 7); /* A7 0 = Disable(Training Mode) */ + lmc_mr_mpr_ctl.cn70xx.mr_wr_addr = mr_wr_addr; + lmc_wr(priv, CVMX_LMCX_MR_MPR_CTL(if_num), lmc_mr_mpr_ctl.u64); +} + +static void set_dram_output_inversion(struct ddr_priv *priv, int if_num, + int dimm_count, int rank_mask, + int inversion) +{ + union cvmx_lmcx_ddr4_dimm_ctl lmc_ddr4_dimm_ctl; + union cvmx_lmcx_dimmx_params lmc_dimmx_params; + union cvmx_lmcx_dimm_ctl lmc_dimm_ctl; + int dimm_no; + + /* Don't touch extenced register control words */ + lmc_ddr4_dimm_ctl.u64 = 0; + lmc_wr(priv, CVMX_LMCX_DDR4_DIMM_CTL(if_num), lmc_ddr4_dimm_ctl.u64); + + debug("All DIMMs: Register Control Word RC0 : %x\n", + (inversion & 1)); + + for (dimm_no = 0; dimm_no < dimm_count; ++dimm_no) { + lmc_dimmx_params.u64 = + lmc_rd(priv, CVMX_LMCX_DIMMX_PARAMS(dimm_no, if_num)); + lmc_dimmx_params.s.rc0 = + (lmc_dimmx_params.s.rc0 & ~1) | (inversion & 1); + + lmc_wr(priv, + CVMX_LMCX_DIMMX_PARAMS(dimm_no, if_num), + lmc_dimmx_params.u64); + } + + /* LMC0_DIMM_CTL */ + lmc_dimm_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DIMM_CTL(if_num)); + lmc_dimm_ctl.s.dimm0_wmask = 0x1; + lmc_dimm_ctl.s.dimm1_wmask = (dimm_count > 1) ? 0x0001 : 0x0000; + + debug("LMC DIMM_CTL : 0x%016llx\n", + lmc_dimm_ctl.u64); + lmc_wr(priv, CVMX_LMCX_DIMM_CTL(if_num), lmc_dimm_ctl.u64); + + oct3_ddr3_seq(priv, rank_mask, if_num, 0x7); /* Init RCW */ +} + +static void write_mpr_page0_pattern(struct ddr_priv *priv, int rank_mask, + int if_num, int dimm_count, int pattern, + int location_mask) +{ + int rankx; + int location; + + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + for (location = 0; location < 4; ++location) { + if (!(location_mask & (1 << location))) + continue; + + ddr4_mpr_write(priv, if_num, rankx, + /* page */ 0, /* location */ location, + pattern); + } + } +} + +static void change_rdimm_mpr_pattern(struct ddr_priv *priv, int rank_mask, + int if_num, int dimm_count) +{ + int save_ref_zqcs_int; + union cvmx_lmcx_config lmc_config; + + /* + * Okay, here is the latest sequence. This should work for all + * chips and passes (78,88,73,etc). This sequence should be run + * immediately after DRAM INIT. The basic idea is to write the + * same pattern into each of the 4 MPR locations in the DRAM, so + * that the same value is returned when doing MPR reads regardless + * of the inversion state. My advice is to put this into a + * function, change_rdimm_mpr_pattern or something like that, so + * that it can be called multiple times, as I think David wants a + * clock-like pattern for OFFSET training, but does not want a + * clock pattern for Bit-Deskew. You should then be able to call + * this at any point in the init sequence (after DRAM init) to + * change the pattern to a new value. + * Mike + * + * A correction: PHY doesn't need any pattern during offset + * training, but needs clock like pattern for internal vref and + * bit-dskew training. So for that reason, these steps below have + * to be conducted before those trainings to pre-condition + * the pattern. David + * + * Note: Step 3, 4, 8 and 9 have to be done through RDIMM + * sequence. If you issue MRW sequence to do RCW write (in o78 pass + * 1 at least), LMC will still do two commands because + * CONTROL[RDIMM_ENA] is still set high. We don't want it to have + * any unintentional mode register write so it's best to do what + * Mike is doing here. + * Andrew + */ + + /* 1) Disable refresh (REF_ZQCS_INT = 0) */ + + debug("1) Disable refresh (REF_ZQCS_INT = 0)\n"); + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + save_ref_zqcs_int = lmc_config.cn78xx.ref_zqcs_int; + lmc_config.cn78xx.ref_zqcs_int = 0; + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64); + + /* + * 2) Put all devices in MPR mode (Run MRW sequence (sequence=8) + * with MODEREG_PARAMS0[MPRLOC]=0, + * MODEREG_PARAMS0[MPR]=1, MR_MPR_CTL[MR_WR_SEL]=3, and + * MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=1) + */ + + debug("2) Put all devices in MPR mode (Run MRW sequence (sequence=8)\n"); + + /* A-side */ + set_mpr_mode(priv, rank_mask, if_num, dimm_count, 1, 0); + /* B-side */ + set_mpr_mode(priv, rank_mask, if_num, dimm_count, 1, 1); + + /* + * a. Or you can set MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=0 and set + * the value you would like directly into + * MR_MPR_CTL[MR_WR_ADDR] + */ + + /* + * 3) Disable RCD Parity (if previously enabled) - parity does not + * work if inversion disabled + */ + + debug("3) Disable RCD Parity\n"); + + /* + * 4) Disable Inversion in the RCD. + * a. I did (3&4) via the RDIMM sequence (seq_sel=7), but it + * may be easier to use the MRW sequence (seq_sel=8). Just set + * MR_MPR_CTL[MR_WR_SEL]=7, MR_MPR_CTL[MR_WR_ADDR][3:0]=data, + * MR_MPR_CTL[MR_WR_ADDR][7:4]=RCD reg + */ + + debug("4) Disable Inversion in the RCD.\n"); + + set_dram_output_inversion(priv, if_num, dimm_count, rank_mask, 1); + + /* + * 5) Disable CONTROL[RDIMM_ENA] so that MR sequence goes out + * non-inverted. + */ + + debug("5) Disable CONTROL[RDIMM_ENA]\n"); + + set_rdimm_mode(priv, if_num, 0); + + /* + * 6) Write all 4 MPR registers with the desired pattern (have to + * do this for all enabled ranks) + * a. MR_MPR_CTL.MPR_WR=1, MR_MPR_CTL.MPR_LOC=0..3, + * MR_MPR_CTL.MR_WR_SEL=0, MR_MPR_CTL.MR_WR_ADDR[7:0]=pattern + */ + + debug("6) Write all 4 MPR page 0 Training Patterns\n"); + + write_mpr_page0_pattern(priv, rank_mask, if_num, dimm_count, 0x55, 0x8); + + /* 7) Re-enable RDIMM_ENA */ + + debug("7) Re-enable RDIMM_ENA\n"); + + set_rdimm_mode(priv, if_num, 1); + + /* 8) Re-enable RDIMM inversion */ + + debug("8) Re-enable RDIMM inversion\n"); + + set_dram_output_inversion(priv, if_num, dimm_count, rank_mask, 0); + + /* 9) Re-enable RDIMM parity (if desired) */ + + debug("9) Re-enable RDIMM parity (if desired)\n"); + + /* + * 10)Take B-side devices out of MPR mode (Run MRW sequence + * (sequence=8) with MODEREG_PARAMS0[MPRLOC]=0, + * MODEREG_PARAMS0[MPR]=0, MR_MPR_CTL[MR_WR_SEL]=3, and + * MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=1) + */ + + debug("10)Take B-side devices out of MPR mode\n"); + + set_mpr_mode(priv, rank_mask, if_num, dimm_count, + /* mpr */ 0, /* bg1 */ 1); + + /* + * a. Or you can set MR_MPR_CTL[MR_WR_USE_DEFAULT_VALUE]=0 and + * set the value you would like directly into MR_MPR_CTL[MR_WR_ADDR] + */ + + /* 11)Re-enable refresh (REF_ZQCS_INT=previous value) */ + + debug("11)Re-enable refresh (REF_ZQCS_INT=previous value)\n"); + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + lmc_config.cn78xx.ref_zqcs_int = save_ref_zqcs_int; + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64); +} + +static int validate_hwl_seq(int *wl, int *seq) +{ + // sequence index, step through the sequence array + int seqx; + int bitnum; + + seqx = 0; + + while (seq[seqx + 1] >= 0) { // stop on next seq entry == -1 + // but now, check current versus next + bitnum = (wl[seq[seqx]] << 2) | wl[seq[seqx + 1]]; + // magic validity number (see matrix above) + if (!((1 << bitnum) & 0xBDE7)) + return 1; + seqx++; + } + + return 0; +} + +static int validate_hw_wl_settings(int if_num, + union cvmx_lmcx_wlevel_rankx + *lmc_wlevel_rank, int is_rdimm, int ecc_ena) +{ + int wl[9], byte, errors; + + // arrange the sequences so + // index 0 has byte 0, etc, ECC in middle + int useq[] = { 0, 1, 2, 3, 8, 4, 5, 6, 7, -1 }; + // index 0 is ECC, then go down + int rseq1[] = { 8, 3, 2, 1, 0, -1 }; + // index 0 has byte 4, then go up + int rseq2[] = { 4, 5, 6, 7, -1 }; + // index 0 has byte 0, etc, no ECC + int useqno[] = { 0, 1, 2, 3, 4, 5, 6, 7, -1 }; + // index 0 is byte 3, then go down, no ECC + int rseq1no[] = { 3, 2, 1, 0, -1 }; + + // in the CSR, bytes 0-7 are always data, byte 8 is ECC + for (byte = 0; byte < (8 + ecc_ena); byte++) { + // preprocess :-) + wl[byte] = (get_wl_rank(lmc_wlevel_rank, byte) >> + 1) & 3; + } + + errors = 0; + if (is_rdimm) { // RDIMM order + errors = validate_hwl_seq(wl, (ecc_ena) ? rseq1 : rseq1no); + errors += validate_hwl_seq(wl, rseq2); + } else { // UDIMM order + errors = validate_hwl_seq(wl, (ecc_ena) ? useq : useqno); + } + + return errors; +} + +static unsigned int extr_wr(u64 u, int x) +{ + return (unsigned int)(((u >> (x * 12 + 5)) & 0x3ULL) | + ((u >> (51 + x - 2)) & 0x4ULL)); +} + +static void insrt_wr(u64 *up, int x, int v) +{ + u64 u = *up; + + u &= ~(((0x3ULL) << (x * 12 + 5)) | ((0x1ULL) << (51 + x))); + *up = (u | ((v & 0x3ULL) << (x * 12 + 5)) | + ((v & 0x4ULL) << (51 + x - 2))); +} + +/* Read out Deskew Settings for DDR */ + +struct deskew_bytes { + u16 bits[8]; +}; + +struct deskew_data { + struct deskew_bytes bytes[9]; +}; + +struct dac_data { + int bytes[9]; +}; + +// T88 pass 1, skip 4=DAC +static const u8 dsk_bit_seq_p1[8] = { 0, 1, 2, 3, 5, 6, 7, 8 }; +// T88 Pass 2, skip 4=DAC and 5=DBI +static const u8 dsk_bit_seq_p2[8] = { 0, 1, 2, 3, 6, 7, 8, 9 }; + +static void get_deskew_settings(struct ddr_priv *priv, int if_num, + struct deskew_data *dskdat) +{ + union cvmx_lmcx_phy_ctl phy_ctl; + union cvmx_lmcx_config lmc_config; + int bit_index; + int byte_lane, byte_limit; + // NOTE: these are for pass 2.x + int is_o78p2 = !octeon_is_cpuid(OCTEON_CN78XX_PASS1_X); + const u8 *bit_seq = (is_o78p2) ? dsk_bit_seq_p2 : dsk_bit_seq_p1; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + byte_limit = ((!lmc_config.s.mode32b) ? 8 : 4) + lmc_config.s.ecc_ena; + + memset(dskdat, 0, sizeof(*dskdat)); + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.dsk_dbg_clk_scaler = 3; + + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) { + phy_ctl.s.dsk_dbg_byte_sel = byte_lane; // set byte lane + + for (bit_index = 0; bit_index < 8; ++bit_index) { + // set bit number and start read sequence + phy_ctl.s.dsk_dbg_bit_sel = bit_seq[bit_index]; + phy_ctl.s.dsk_dbg_rd_start = 1; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + // poll for read sequence to complete + do { + phy_ctl.u64 = + lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + } while (phy_ctl.s.dsk_dbg_rd_complete != 1); + + // record the data + dskdat->bytes[byte_lane].bits[bit_index] = + phy_ctl.s.dsk_dbg_rd_data & 0x3ff; + } + } +} + +static void display_deskew_settings(struct ddr_priv *priv, int if_num, + struct deskew_data *dskdat, + int print_enable) +{ + int byte_lane; + int bit_num; + u16 flags, deskew; + union cvmx_lmcx_config lmc_config; + int byte_limit; + const char *fc = " ?-=+*#&"; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena; + + if (print_enable) { + debug("N0.LMC%d: Deskew Data: Bit => :", + if_num); + for (bit_num = 7; bit_num >= 0; --bit_num) + debug(" %3d ", bit_num); + debug("\n"); + } + + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) { + if (print_enable) + debug("N0.LMC%d: Bit Deskew Byte %d %s :", + if_num, byte_lane, + (print_enable >= 3) ? "FINAL" : " "); + + for (bit_num = 7; bit_num >= 0; --bit_num) { + flags = dskdat->bytes[byte_lane].bits[bit_num] & 7; + deskew = dskdat->bytes[byte_lane].bits[bit_num] >> 3; + + if (print_enable) + debug(" %3d %c", deskew, fc[flags ^ 1]); + + } /* for (bit_num = 7; bit_num >= 0; --bit_num) */ + + if (print_enable) + debug("\n"); + } +} + +#if ALLOW_BY_RANK_INIT +static void override_deskew_settings(struct ddr_priv *priv, int if_num, + struct deskew_data *dskdat) +{ + union cvmx_lmcx_phy_ctl phy_ctl; + union cvmx_lmcx_config lmc_config; + + int bit, byte_lane, byte_limit; + u64 csr_data; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena; + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + + phy_ctl.s.phy_reset = 0; + phy_ctl.s.dsk_dbg_num_bits_sel = 1; + phy_ctl.s.dsk_dbg_offset = 0; + phy_ctl.s.dsk_dbg_clk_scaler = 3; + + phy_ctl.s.dsk_dbg_wr_mode = 1; + phy_ctl.s.dsk_dbg_load_dis = 0; + phy_ctl.s.dsk_dbg_overwrt_ena = 0; + + phy_ctl.s.phy_dsk_reset = 0; + + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) { + csr_data = 0; + // FIXME: can we ignore DBI? + for (bit = 0; bit < 8; ++bit) { + // fetch input and adjust + u64 bits = (dskdat->bytes[byte_lane].bits[bit] >> 3) & + 0x7F; + + /* + * lmc_general_purpose0.data[6:0] // DQ0 + * lmc_general_purpose0.data[13:7] // DQ1 + * lmc_general_purpose0.data[20:14] // DQ2 + * lmc_general_purpose0.data[27:21] // DQ3 + * lmc_general_purpose0.data[34:28] // DQ4 + * lmc_general_purpose0.data[41:35] // DQ5 + * lmc_general_purpose0.data[48:42] // DQ6 + * lmc_general_purpose0.data[55:49] // DQ7 + * lmc_general_purpose0.data[62:56] // DBI + */ + csr_data |= (bits << (7 * bit)); + + } /* for (bit = 0; bit < 8; ++bit) */ + + // update GP0 with the bit data for this byte lane + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE0(if_num), csr_data); + lmc_rd(priv, CVMX_LMCX_GENERAL_PURPOSE0(if_num)); + + // start the deskew load sequence + phy_ctl.s.dsk_dbg_byte_sel = byte_lane; + phy_ctl.s.dsk_dbg_rd_start = 1; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + // poll for read sequence to complete + do { + udelay(100); + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + } while (phy_ctl.s.dsk_dbg_rd_complete != 1); + } + + // tell phy to use the new settings + phy_ctl.s.dsk_dbg_overwrt_ena = 1; + phy_ctl.s.dsk_dbg_rd_start = 0; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + phy_ctl.s.dsk_dbg_wr_mode = 0; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); +} + +static void process_by_rank_dac(struct ddr_priv *priv, int if_num, + int rank_mask, struct dac_data *dacdat) +{ + union cvmx_lmcx_config lmc_config; + int rankx, byte_lane; + int byte_limit; + int rank_count; + struct dac_data dacsum; + int lane_probs; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena; + + memset((void *)&dacsum, 0, sizeof(dacsum)); + rank_count = 0; + lane_probs = 0; + + for (rankx = 0; rankx < 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + rank_count++; + + display_dac_dbi_settings(if_num, /*dac */ 1, + lmc_config.s.ecc_ena, + &dacdat[rankx].bytes[0], + "By-Ranks VREF"); + // sum + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) { + if (rank_count == 2) { + int ranks_diff = + abs((dacsum.bytes[byte_lane] - + dacdat[rankx].bytes[byte_lane])); + + // FIXME: is 19 a good number? + if (ranks_diff > 19) + lane_probs |= (1 << byte_lane); + } + dacsum.bytes[byte_lane] += + dacdat[rankx].bytes[byte_lane]; + } + } + + // average + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) + dacsum.bytes[byte_lane] /= rank_count; // FIXME: nint? + + display_dac_dbi_settings(if_num, /*dac */ 1, lmc_config.s.ecc_ena, + &dacsum.bytes[0], "All-Rank VREF"); + + if (lane_probs) { + debug("N0.LMC%d: All-Rank VREF DAC Problem Bytelane(s): 0x%03x\n", + if_num, lane_probs); + } + + // finally, write the averaged DAC values + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) { + load_dac_override(priv, if_num, dacsum.bytes[byte_lane], + byte_lane); + } +} + +static void process_by_rank_dsk(struct ddr_priv *priv, int if_num, + int rank_mask, struct deskew_data *dskdat) +{ + union cvmx_lmcx_config lmc_config; + int rankx, lane, bit; + int byte_limit; + struct deskew_data dsksum, dskcnt; + u16 deskew; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + byte_limit = ((lmc_config.s.mode32b) ? 4 : 8) + lmc_config.s.ecc_ena; + + memset((void *)&dsksum, 0, sizeof(dsksum)); + memset((void *)&dskcnt, 0, sizeof(dskcnt)); + + for (rankx = 0; rankx < 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + // sum ranks + for (lane = 0; lane < byte_limit; lane++) { + for (bit = 0; bit < 8; ++bit) { + deskew = dskdat[rankx].bytes[lane].bits[bit]; + // if flags indicate sat hi or lo, skip it + if (deskew & 6) + continue; + + // clear flags + dsksum.bytes[lane].bits[bit] += + deskew & ~7; + // count entries + dskcnt.bytes[lane].bits[bit] += 1; + } + } + } + + // average ranks + for (lane = 0; lane < byte_limit; lane++) { + for (bit = 0; bit < 8; ++bit) { + int div = dskcnt.bytes[lane].bits[bit]; + + if (div > 0) { + dsksum.bytes[lane].bits[bit] /= div; + // clear flags + dsksum.bytes[lane].bits[bit] &= ~7; + // set LOCK + dsksum.bytes[lane].bits[bit] |= 1; + } else { + // FIXME? use reset value? + dsksum.bytes[lane].bits[bit] = + (64 << 3) | 1; + } + } + } + + // TME for FINAL version + display_deskew_settings(priv, if_num, &dsksum, /*VBL_TME */ 3); + + // finally, write the averaged DESKEW values + override_deskew_settings(priv, if_num, &dsksum); +} +#endif /* ALLOW_BY_RANK_INIT */ + +struct deskew_counts { + int saturated; // number saturated + int unlocked; // number unlocked + int nibrng_errs; // nibble range errors + int nibunl_errs; // nibble unlocked errors + int bitval_errs; // bit value errors +}; + +#define MIN_BITVAL 17 +#define MAX_BITVAL 110 + +static void validate_deskew_training(struct ddr_priv *priv, int rank_mask, + int if_num, struct deskew_counts *counts, + int print_flags) +{ + int byte_lane, bit_index, nib_num; + int nibrng_errs, nibunl_errs, bitval_errs; + union cvmx_lmcx_config lmc_config; + s16 nib_min[2], nib_max[2], nib_unl[2]; + int byte_limit; + int print_enable = print_flags & 1; + struct deskew_data dskdat; + s16 flags, deskew; + const char *fc = " ?-=+*#&"; + int bit_last; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + byte_limit = ((!lmc_config.s.mode32b) ? 8 : 4) + lmc_config.s.ecc_ena; + + memset(counts, 0, sizeof(struct deskew_counts)); + + get_deskew_settings(priv, if_num, &dskdat); + + if (print_enable) { + debug("N0.LMC%d: Deskew Settings: Bit => :", + if_num); + for (bit_index = 7; bit_index >= 0; --bit_index) + debug(" %3d ", bit_index); + debug("\n"); + } + + for (byte_lane = 0; byte_lane < byte_limit; byte_lane++) { + if (print_enable) + debug("N0.LMC%d: Bit Deskew Byte %d %s :", + if_num, byte_lane, + (print_flags & 2) ? "FINAL" : " "); + + nib_min[0] = 127; + nib_min[1] = 127; + nib_max[0] = 0; + nib_max[1] = 0; + nib_unl[0] = 0; + nib_unl[1] = 0; + + if (lmc_config.s.mode32b == 1 && byte_lane == 4) { + bit_last = 3; + if (print_enable) + debug(" "); + } else { + bit_last = 7; + } + + for (bit_index = bit_last; bit_index >= 0; --bit_index) { + nib_num = (bit_index > 3) ? 1 : 0; + + flags = dskdat.bytes[byte_lane].bits[bit_index] & 7; + deskew = dskdat.bytes[byte_lane].bits[bit_index] >> 3; + + counts->saturated += !!(flags & 6); + + // Do range calc even when locked; it could happen + // that a bit is still unlocked after final retry, + // and we want to have an external retry if a RANGE + // error is present at exit... + nib_min[nib_num] = min(nib_min[nib_num], deskew); + nib_max[nib_num] = max(nib_max[nib_num], deskew); + + if (!(flags & 1)) { // only when not locked + counts->unlocked += 1; + nib_unl[nib_num] += 1; + } + + if (print_enable) + debug(" %3d %c", deskew, fc[flags ^ 1]); + } + + /* + * Now look for nibble errors + * + * For bit 55, it looks like a bit deskew problem. When the + * upper nibble of byte 6 needs to go to saturation, bit 7 + * of byte 6 locks prematurely at 64. For DIMMs with raw + * card A and B, can we reset the deskew training when we + * encounter this case? The reset criteria should be looking + * at one nibble at a time for raw card A and B; if the + * bit-deskew setting within a nibble is different by > 33, + * we'll issue a reset to the bit deskew training. + * + * LMC0 Bit Deskew Byte(6): 64 0 - 0 - 0 - 26 61 35 64 + */ + // upper nibble range, then lower nibble range + nibrng_errs = ((nib_max[1] - nib_min[1]) > 33) ? 1 : 0; + nibrng_errs |= ((nib_max[0] - nib_min[0]) > 33) ? 1 : 0; + + // check for nibble all unlocked + nibunl_errs = ((nib_unl[0] == 4) || (nib_unl[1] == 4)) ? 1 : 0; + + // check for bit value errors, ie < 17 or > 110 + // FIXME? assume max always > MIN_BITVAL and min < MAX_BITVAL + bitval_errs = ((nib_max[1] > MAX_BITVAL) || + (nib_max[0] > MAX_BITVAL)) ? 1 : 0; + bitval_errs |= ((nib_min[1] < MIN_BITVAL) || + (nib_min[0] < MIN_BITVAL)) ? 1 : 0; + + if ((nibrng_errs != 0 || nibunl_errs != 0 || + bitval_errs != 0) && print_enable) { + debug(" %c%c%c", + (nibrng_errs) ? 'R' : ' ', + (nibunl_errs) ? 'U' : ' ', + (bitval_errs) ? 'V' : ' '); + } + + if (print_enable) + debug("\n"); + + counts->nibrng_errs |= (nibrng_errs << byte_lane); + counts->nibunl_errs |= (nibunl_errs << byte_lane); + counts->bitval_errs |= (bitval_errs << byte_lane); + } +} + +static unsigned short load_dac_override(struct ddr_priv *priv, int if_num, + int dac_value, int byte) +{ + union cvmx_lmcx_dll_ctl3 ddr_dll_ctl3; + // single bytelanes incr by 1; A is for ALL + int bytex = (byte == 0x0A) ? byte : byte + 1; + + ddr_dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); + + SET_DDR_DLL_CTL3(byte_sel, bytex); + SET_DDR_DLL_CTL3(offset, dac_value >> 1); + + ddr_dll_ctl3.cn73xx.bit_select = 0x9; /* No-op */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + ddr_dll_ctl3.cn73xx.bit_select = 0xC; /* vref bypass setting load */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + ddr_dll_ctl3.cn73xx.bit_select = 0xD; /* vref bypass on. */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + ddr_dll_ctl3.cn73xx.bit_select = 0x9; /* No-op */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); // flush writes + + return (unsigned short)GET_DDR_DLL_CTL3(offset); +} + +// arg dac_or_dbi is 1 for DAC, 0 for DBI +// returns 9 entries (bytelanes 0 through 8) in settings[] +// returns 0 if OK, -1 if a problem +static int read_dac_dbi_settings(struct ddr_priv *priv, int if_num, + int dac_or_dbi, int *settings) +{ + union cvmx_lmcx_phy_ctl phy_ctl; + int byte_lane, bit_num; + int deskew; + int dac_value; + int new_deskew_layout = 0; + + new_deskew_layout = octeon_is_cpuid(OCTEON_CN73XX) || + octeon_is_cpuid(OCTEON_CNF75XX); + new_deskew_layout |= (octeon_is_cpuid(OCTEON_CN78XX) && + !octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)); + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.dsk_dbg_clk_scaler = 3; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + bit_num = (dac_or_dbi) ? 4 : 5; + // DBI not available + if (bit_num == 5 && !new_deskew_layout) + return -1; + + // FIXME: always assume ECC is available + for (byte_lane = 8; byte_lane >= 0; --byte_lane) { + //set byte lane and bit to read + phy_ctl.s.dsk_dbg_bit_sel = bit_num; + phy_ctl.s.dsk_dbg_byte_sel = byte_lane; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + //start read sequence + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.dsk_dbg_rd_start = 1; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + //poll for read sequence to complete + do { + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + } while (phy_ctl.s.dsk_dbg_rd_complete != 1); + + // keep the flag bits where they are for DBI + deskew = phy_ctl.s.dsk_dbg_rd_data; /* >> 3 */ + dac_value = phy_ctl.s.dsk_dbg_rd_data & 0xff; + + settings[byte_lane] = (dac_or_dbi) ? dac_value : deskew; + } + + return 0; +} + +// print out the DBI settings array +// arg dac_or_dbi is 1 for DAC, 0 for DBI +static void display_dac_dbi_settings(int lmc, int dac_or_dbi, + int ecc_ena, int *settings, char *title) +{ + int byte; + int flags; + int deskew; + const char *fc = " ?-=+*#&"; + + debug("N0.LMC%d: %s %s Settings %d:0 :", + lmc, title, (dac_or_dbi) ? "DAC" : "DBI", 7 + ecc_ena); + // FIXME: what about 32-bit mode? + for (byte = (7 + ecc_ena); byte >= 0; --byte) { + if (dac_or_dbi) { // DAC + flags = 1; // say its locked to get blank + deskew = settings[byte] & 0xff; + } else { // DBI + flags = settings[byte] & 7; + deskew = (settings[byte] >> 3) & 0x7f; + } + debug(" %3d %c", deskew, fc[flags ^ 1]); + } + debug("\n"); +} + +// Find a HWL majority +static int find_wl_majority(struct wlevel_bitcnt *bc, int *mx, int *mc, + int *xc, int *cc) +{ + int ix, ic; + + *mx = -1; + *mc = 0; + *xc = 0; + *cc = 0; + + for (ix = 0; ix < 4; ix++) { + ic = bc->bitcnt[ix]; + + // make a bitmask of the ones with a count + if (ic > 0) { + *mc |= (1 << ix); + *cc += 1; // count how many had non-zero counts + } + + // find the majority + if (ic > *xc) { // new max? + *xc = ic; // yes + *mx = ix; // set its index + } + } + + return (*mx << 1); +} + +// Evaluate the DAC settings array +static int evaluate_dac_settings(int if_64b, int ecc_ena, int *settings) +{ + int byte, lane, dac, comp; + int last = (if_64b) ? 7 : 3; + + // FIXME: change the check...??? + // this looks only for sets of DAC values whose max/min differ by a lot + // let any EVEN go so long as it is within range... + for (byte = (last + ecc_ena); byte >= 0; --byte) { + dac = settings[byte] & 0xff; + + for (lane = (last + ecc_ena); lane >= 0; --lane) { + comp = settings[lane] & 0xff; + if (abs((dac - comp)) > 25) + return 1; + } + } + + return 0; +} + +static void perform_offset_training(struct ddr_priv *priv, int rank_mask, + int if_num) +{ + union cvmx_lmcx_phy_ctl lmc_phy_ctl; + u64 orig_phy_ctl; + const char *s; + + /* + * 4.8.6 LMC Offset Training + * + * LMC requires input-receiver offset training. + * + * 1. Write LMC(0)_PHY_CTL[DAC_ON] = 1 + */ + lmc_phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + orig_phy_ctl = lmc_phy_ctl.u64; + lmc_phy_ctl.s.dac_on = 1; + + // allow full CSR override + s = lookup_env_ull(priv, "ddr_phy_ctl"); + if (s) + lmc_phy_ctl.u64 = strtoull(s, NULL, 0); + + // do not print or write if CSR does not change... + if (lmc_phy_ctl.u64 != orig_phy_ctl) { + debug("PHY_CTL : 0x%016llx\n", + lmc_phy_ctl.u64); + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), lmc_phy_ctl.u64); + } + + /* + * 2. Write LMC(0)_SEQ_CTL[SEQ_SEL] = 0x0B and + * LMC(0)_SEQ_CTL[INIT_START] = 1. + * + * 3. Wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] to be set to 1. + */ + /* Start Offset training sequence */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x0B); +} + +static void perform_internal_vref_training(struct ddr_priv *priv, + int rank_mask, int if_num) +{ + union cvmx_lmcx_ext_config ext_config; + union cvmx_lmcx_dll_ctl3 ddr_dll_ctl3; + + // First, make sure all byte-lanes are out of VREF bypass mode + ddr_dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); + + ddr_dll_ctl3.cn78xx.byte_sel = 0x0A; /* all byte-lanes */ + ddr_dll_ctl3.cn78xx.bit_select = 0x09; /* No-op */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + ddr_dll_ctl3.cn78xx.bit_select = 0x0E; /* vref bypass off. */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + ddr_dll_ctl3.cn78xx.bit_select = 0x09; /* No-op */ + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + + /* + * 4.8.7 LMC Internal vref Training + * + * LMC requires input-reference-voltage training. + * + * 1. Write LMC(0)_EXT_CONFIG[VREFINT_SEQ_DESKEW] = 0. + */ + ext_config.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num)); + ext_config.s.vrefint_seq_deskew = 0; + + ddr_seq_print("Performing LMC sequence: vrefint_seq_deskew = %d\n", + ext_config.s.vrefint_seq_deskew); + + lmc_wr(priv, CVMX_LMCX_EXT_CONFIG(if_num), ext_config.u64); + + /* + * 2. Write LMC(0)_SEQ_CTL[SEQ_SEL] = 0x0a and + * LMC(0)_SEQ_CTL[INIT_START] = 1. + * + * 3. Wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] to be set to 1. + */ + /* Start LMC Internal vref Training */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x0A); +} + +#define dbg_avg(format, ...) // debug(format, ##__VA_ARGS__) + +static int process_samples_average(s16 *bytes, int num_samples, + int lmc, int lane_no) +{ + int i, sadj, sum = 0, ret, asum, trunc; + s16 smin = 32767, smax = -32768; + int nmin, nmax; + //int rng; + + dbg_avg("DBG_AVG%d.%d: ", lmc, lane_no); + + for (i = 0; i < num_samples; i++) { + sum += bytes[i]; + if (bytes[i] < smin) + smin = bytes[i]; + if (bytes[i] > smax) + smax = bytes[i]; + dbg_avg(" %3d", bytes[i]); + } + + nmin = 0; + nmax = 0; + for (i = 0; i < num_samples; i++) { + if (bytes[i] == smin) + nmin += 1; + if (bytes[i] == smax) + nmax += 1; + } + dbg_avg(" (min=%3d/%d, max=%3d/%d, range=%2d, samples=%2d)", + smin, nmin, smax, nmax, rng, num_samples); + + asum = sum - smin - smax; + + sadj = divide_nint(asum * 10, (num_samples - 2)); + + trunc = asum / (num_samples - 2); + + dbg_avg(" [%3d.%d, %3d]", sadj / 10, sadj % 10, trunc); + + sadj = divide_nint(sadj, 10); + if (trunc & 1) + ret = trunc; + else if (sadj & 1) + ret = sadj; + else + ret = trunc + 1; + + dbg_avg(" -> %3d\n", ret); + + return ret; +} + +#define DEFAULT_SAT_RETRY_LIMIT 11 // 1 + 10 retries + +#define default_lock_retry_limit 20 // 20 retries +#define deskew_validation_delay 10000 // 10 millisecs + +static int perform_deskew_training(struct ddr_priv *priv, int rank_mask, + int if_num, int spd_rawcard_aorb) +{ + int unsaturated, locked; + int sat_retries, sat_retries_limit; + int lock_retries, lock_retries_total, lock_retries_limit; + int print_first; + int print_them_all; + struct deskew_counts dsk_counts; + union cvmx_lmcx_phy_ctl phy_ctl; + char *s; + int has_no_sat = octeon_is_cpuid(OCTEON_CN78XX_PASS2_X) || + octeon_is_cpuid(OCTEON_CNF75XX); + int disable_bitval_retries = 1; // default to disabled + + debug("N0.LMC%d: Performing Deskew Training.\n", if_num); + + sat_retries = 0; + sat_retries_limit = (has_no_sat) ? 5 : DEFAULT_SAT_RETRY_LIMIT; + + lock_retries_total = 0; + unsaturated = 0; + print_first = 1; // print the first one + // set to true for printing all normal deskew attempts + print_them_all = 0; + + // provide override for bitval_errs causing internal VREF retries + s = env_get("ddr_disable_bitval_retries"); + if (s) + disable_bitval_retries = !!simple_strtoul(s, NULL, 0); + + lock_retries_limit = default_lock_retry_limit; + if ((octeon_is_cpuid(OCTEON_CN78XX_PASS2_X)) || + (octeon_is_cpuid(OCTEON_CN73XX)) || + (octeon_is_cpuid(OCTEON_CNF75XX))) + lock_retries_limit *= 2; // give new chips twice as many + + do { /* while (sat_retries < sat_retry_limit) */ + /* + * 4.8.8 LMC Deskew Training + * + * LMC requires input-read-data deskew training. + * + * 1. Write LMC(0)_EXT_CONFIG[VREFINT_SEQ_DESKEW] = 1. + */ + + union cvmx_lmcx_ext_config ext_config; + + ext_config.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num)); + ext_config.s.vrefint_seq_deskew = 1; + + ddr_seq_print + ("Performing LMC sequence: vrefint_seq_deskew = %d\n", + ext_config.s.vrefint_seq_deskew); + + lmc_wr(priv, CVMX_LMCX_EXT_CONFIG(if_num), ext_config.u64); + + /* + * 2. Write LMC(0)_SEQ_CTL[SEQ_SEL] = 0x0A and + * LMC(0)_SEQ_CTL[INIT_START] = 1. + * + * 3. Wait for LMC(0)_SEQ_CTL[SEQ_COMPLETE] to be set to 1. + */ + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.phy_dsk_reset = 1; /* RESET Deskew sequence */ + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + /* LMC Deskew Training */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x0A); + + lock_retries = 0; + +perform_deskew_training: + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.phy_dsk_reset = 0; /* Normal Deskew sequence */ + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + /* LMC Deskew Training */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x0A); + + // Moved this from validate_deskew_training + /* Allow deskew results to stabilize before evaluating them. */ + udelay(deskew_validation_delay); + + // Now go look at lock and saturation status... + validate_deskew_training(priv, rank_mask, if_num, &dsk_counts, + print_first); + // after printing the first and not doing them all, no more + if (print_first && !print_them_all) + print_first = 0; + + unsaturated = (dsk_counts.saturated == 0); + locked = (dsk_counts.unlocked == 0); + + // only do locking retries if unsaturated or rawcard A or B, + // otherwise full SAT retry + if (unsaturated || (spd_rawcard_aorb && !has_no_sat)) { + if (!locked) { // and not locked + lock_retries++; + lock_retries_total++; + if (lock_retries <= lock_retries_limit) { + goto perform_deskew_training; + } else { + debug("N0.LMC%d: LOCK RETRIES failed after %d retries\n", + if_num, lock_retries_limit); + } + } else { + // only print if we did try + if (lock_retries_total > 0) + debug("N0.LMC%d: LOCK RETRIES successful after %d retries\n", + if_num, lock_retries); + } + } /* if (unsaturated || spd_rawcard_aorb) */ + + ++sat_retries; + + /* + * At this point, check for a DDR4 RDIMM that will not + * benefit from SAT retries; if so, exit + */ + if (spd_rawcard_aorb && !has_no_sat) { + debug("N0.LMC%d: Deskew Training Loop: Exiting for RAWCARD == A or B.\n", + if_num); + break; // no sat or lock retries + } + + } while (!unsaturated && (sat_retries < sat_retries_limit)); + + debug("N0.LMC%d: Deskew Training %s. %d sat-retries, %d lock-retries\n", + if_num, (sat_retries >= DEFAULT_SAT_RETRY_LIMIT) ? + "Timed Out" : "Completed", sat_retries - 1, lock_retries_total); + + // FIXME? add saturation to reasons for fault return - give it a + // chance via Internal VREF + // FIXME? add OPTIONAL bit value to reasons for fault return - + // give it a chance via Internal VREF + if (dsk_counts.nibrng_errs != 0 || dsk_counts.nibunl_errs != 0 || + (dsk_counts.bitval_errs != 0 && !disable_bitval_retries) || + !unsaturated) { + debug("N0.LMC%d: Nibble or Saturation Error(s) found, returning FAULT\n", + if_num); + // FIXME: do we want this output always for errors? + validate_deskew_training(priv, rank_mask, if_num, + &dsk_counts, 1); + return -1; // we did retry locally, they did not help + } + + // NOTE: we (currently) always print one last training validation + // before starting Read Leveling... + + return 0; +} + +#define SCALING_FACTOR (1000) + +// NOTE: this gets called for 1-rank and 2-rank DIMMs in single-slot config +static int compute_vref_1slot_2rank(int rtt_wr, int rtt_park, int dqx_ctl, + int rank_count, int dram_connection) +{ + u64 reff_s; + u64 rser_s = (dram_connection) ? 0 : 15; + u64 vdd = 1200; + u64 vref; + // 99 == HiZ + u64 rtt_wr_s = (((rtt_wr == 0) || rtt_wr == 99) ? + 1 * 1024 * 1024 : rtt_wr); + u64 rtt_park_s = (((rtt_park == 0) || ((rank_count == 1) && + (rtt_wr != 0))) ? + 1 * 1024 * 1024 : rtt_park); + u64 dqx_ctl_s = (dqx_ctl == 0 ? 1 * 1024 * 1024 : dqx_ctl); + int vref_value; + u64 rangepc = 6000; // range1 base + u64 vrefpc; + int vref_range = 0; + + reff_s = divide_nint((rtt_wr_s * rtt_park_s), (rtt_wr_s + rtt_park_s)); + + vref = (((rser_s + dqx_ctl_s) * SCALING_FACTOR) / + (rser_s + dqx_ctl_s + reff_s)) + SCALING_FACTOR; + + vref = (vref * vdd) / 2 / SCALING_FACTOR; + + vrefpc = (vref * 100 * 100) / vdd; + + if (vrefpc < rangepc) { // < range1 base, use range2 + vref_range = 1 << 6; // set bit A6 for range2 + rangepc = 4500; // range2 base is 45% + } + + vref_value = divide_nint(vrefpc - rangepc, 65); + if (vref_value < 0) + vref_value = vref_range; // set to base of range + else + vref_value |= vref_range; + + debug("rtt_wr: %d, rtt_park: %d, dqx_ctl: %d, rank_count: %d\n", + rtt_wr, rtt_park, dqx_ctl, rank_count); + debug("rtt_wr_s: %lld, rtt_park_s: %lld, dqx_ctl_s: %lld, vref_value: 0x%x, range: %d\n", + rtt_wr_s, rtt_park_s, dqx_ctl_s, vref_value ^ vref_range, + vref_range ? 2 : 1); + + return vref_value; +} + +// NOTE: this gets called for 1-rank and 2-rank DIMMs in two-slot configs +static int compute_vref_2slot_2rank(int rtt_wr, int rtt_park_00, + int rtt_park_01, + int dqx_ctl, int rtt_nom, + int dram_connection) +{ + u64 rser = (dram_connection) ? 0 : 15; + u64 vdd = 1200; + u64 vl, vlp, vcm; + u64 rd0, rd1, rpullup; + // 99 == HiZ + u64 rtt_wr_s = (((rtt_wr == 0) || rtt_wr == 99) ? + 1 * 1024 * 1024 : rtt_wr); + u64 rtt_park_00_s = (rtt_park_00 == 0 ? 1 * 1024 * 1024 : rtt_park_00); + u64 rtt_park_01_s = (rtt_park_01 == 0 ? 1 * 1024 * 1024 : rtt_park_01); + u64 dqx_ctl_s = (dqx_ctl == 0 ? 1 * 1024 * 1024 : dqx_ctl); + u64 rtt_nom_s = (rtt_nom == 0 ? 1 * 1024 * 1024 : rtt_nom); + int vref_value; + u64 rangepc = 6000; // range1 base + u64 vrefpc; + int vref_range = 0; + + // rd0 = (RTT_NOM (parallel) RTT_WR) + = + // ((RTT_NOM * RTT_WR) / (RTT_NOM + RTT_WR)) + RSER + rd0 = divide_nint((rtt_nom_s * rtt_wr_s), + (rtt_nom_s + rtt_wr_s)) + rser; + + // rd1 = (RTT_PARK_00 (parallel) RTT_PARK_01) + RSER = + // ((RTT_PARK_00 * RTT_PARK_01) / (RTT_PARK_00 + RTT_PARK_01)) + RSER + rd1 = divide_nint((rtt_park_00_s * rtt_park_01_s), + (rtt_park_00_s + rtt_park_01_s)) + rser; + + // rpullup = rd0 (parallel) rd1 = (rd0 * rd1) / (rd0 + rd1) + rpullup = divide_nint((rd0 * rd1), (rd0 + rd1)); + + // vl = (DQX_CTL / (DQX_CTL + rpullup)) * 1.2 + vl = divide_nint((dqx_ctl_s * vdd), (dqx_ctl_s + rpullup)); + + // vlp = ((RSER / rd0) * (1.2 - vl)) + vl + vlp = divide_nint((rser * (vdd - vl)), rd0) + vl; + + // vcm = (vlp + 1.2) / 2 + vcm = divide_nint((vlp + vdd), 2); + + // vrefpc = (vcm / 1.2) * 100 + vrefpc = divide_nint((vcm * 100 * 100), vdd); + + if (vrefpc < rangepc) { // < range1 base, use range2 + vref_range = 1 << 6; // set bit A6 for range2 + rangepc = 4500; // range2 base is 45% + } + + vref_value = divide_nint(vrefpc - rangepc, 65); + if (vref_value < 0) + vref_value = vref_range; // set to base of range + else + vref_value |= vref_range; + + debug("rtt_wr:%d, rtt_park_00:%d, rtt_park_01:%d, dqx_ctl:%d, rtt_nom:%d, vref_value:%d (0x%x)\n", + rtt_wr, rtt_park_00, rtt_park_01, dqx_ctl, rtt_nom, vref_value, + vref_value); + + return vref_value; +} + +// NOTE: only call this for DIMMs with 1 or 2 ranks, not 4. +static int compute_vref_val(struct ddr_priv *priv, int if_num, int rankx, + int dimm_count, int rank_count, + struct impedence_values *imp_values, + int is_stacked_die, int dram_connection) +{ + int computed_final_vref_value = 0; + int enable_adjust = ENABLE_COMPUTED_VREF_ADJUSTMENT; + const char *s; + int rtt_wr, dqx_ctl, rtt_nom, index; + union cvmx_lmcx_modereg_params1 lmc_modereg_params1; + union cvmx_lmcx_modereg_params2 lmc_modereg_params2; + union cvmx_lmcx_comp_ctl2 comp_ctl2; + int rtt_park; + int rtt_park_00; + int rtt_park_01; + + debug("N0.LMC%d.R%d: %s(...dram_connection = %d)\n", + if_num, rankx, __func__, dram_connection); + + // allow some overrides... + s = env_get("ddr_adjust_computed_vref"); + if (s) { + enable_adjust = !!simple_strtoul(s, NULL, 0); + if (!enable_adjust) { + debug("N0.LMC%d.R%d: DISABLE adjustment of computed VREF\n", + if_num, rankx); + } + } + + s = env_get("ddr_set_computed_vref"); + if (s) { + int new_vref = simple_strtoul(s, NULL, 0); + + debug("N0.LMC%d.R%d: OVERRIDE computed VREF to 0x%x (%d)\n", + if_num, rankx, new_vref, new_vref); + return new_vref; + } + + /* + * Calculate an alternative to the measured vref value + * but only for configurations we know how to... + */ + // We have code for 2-rank DIMMs in both 1-slot or 2-slot configs, + // and can use the 2-rank 1-slot code for 1-rank DIMMs in 1-slot + // configs, and can use the 2-rank 2-slot code for 1-rank DIMMs + // in 2-slot configs. + + lmc_modereg_params1.u64 = + lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num)); + lmc_modereg_params2.u64 = + lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS2(if_num)); + comp_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + dqx_ctl = imp_values->dqx_strength[comp_ctl2.s.dqx_ctl]; + + // WR always comes from the current rank + index = (lmc_modereg_params1.u64 >> (rankx * 12 + 5)) & 0x03; + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) + index |= lmc_modereg_params1.u64 >> (51 + rankx - 2) & 0x04; + rtt_wr = imp_values->rtt_wr_ohms[index]; + + // separate calculations for 1 vs 2 DIMMs per LMC + if (dimm_count == 1) { + // PARK comes from this rank if 1-rank, otherwise other rank + index = + (lmc_modereg_params2.u64 >> + ((rankx ^ (rank_count - 1)) * 10 + 0)) & 0x07; + rtt_park = imp_values->rtt_nom_ohms[index]; + computed_final_vref_value = + compute_vref_1slot_2rank(rtt_wr, rtt_park, dqx_ctl, + rank_count, dram_connection); + } else { + // get both PARK values from the other DIMM + index = + (lmc_modereg_params2.u64 >> ((rankx ^ 0x02) * 10 + 0)) & + 0x07; + rtt_park_00 = imp_values->rtt_nom_ohms[index]; + index = + (lmc_modereg_params2.u64 >> ((rankx ^ 0x03) * 10 + 0)) & + 0x07; + rtt_park_01 = imp_values->rtt_nom_ohms[index]; + // NOM comes from this rank if 1-rank, otherwise other rank + index = + (lmc_modereg_params1.u64 >> + ((rankx ^ (rank_count - 1)) * 12 + 9)) & 0x07; + rtt_nom = imp_values->rtt_nom_ohms[index]; + computed_final_vref_value = + compute_vref_2slot_2rank(rtt_wr, rtt_park_00, rtt_park_01, + dqx_ctl, rtt_nom, dram_connection); + } + + if (enable_adjust) { + union cvmx_lmcx_config lmc_config; + union cvmx_lmcx_control lmc_control; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + lmc_control.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + + /* + * New computed vref = existing computed vref – X + * + * The value of X is depending on different conditions. + * Both #122 and #139 are 2Rx4 RDIMM, while #124 is stacked + * die 2Rx4, so I conclude the results into two conditions: + * + * 1. Stacked Die: 2Rx4 + * 1-slot: offset = 7. i, e New computed vref = existing + * computed vref – 7 + * 2-slot: offset = 6 + * + * 2. Regular: 2Rx4 + * 1-slot: offset = 3 + * 2-slot: offset = 2 + */ + // we know we never get called unless DDR4, so test just + // the other conditions + if (lmc_control.s.rdimm_ena == 1 && + rank_count == 2 && lmc_config.s.mode_x4dev) { + // it must first be RDIMM and 2-rank and x4 + int adj; + + // now do according to stacked die or not... + if (is_stacked_die) + adj = (dimm_count == 1) ? -7 : -6; + else + adj = (dimm_count == 1) ? -3 : -2; + + // we must have adjusted it, so print it out if + // verbosity is right + debug("N0.LMC%d.R%d: adjusting computed vref from %2d (0x%02x) to %2d (0x%02x)\n", + if_num, rankx, computed_final_vref_value, + computed_final_vref_value, + computed_final_vref_value + adj, + computed_final_vref_value + adj); + computed_final_vref_value += adj; + } + } + + return computed_final_vref_value; +} + +static void unpack_rlevel_settings(int if_bytemask, int ecc_ena, + struct rlevel_byte_data *rlevel_byte, + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank) +{ + if ((if_bytemask & 0xff) == 0xff) { + if (ecc_ena) { + rlevel_byte[8].delay = lmc_rlevel_rank.s.byte7; + rlevel_byte[7].delay = lmc_rlevel_rank.s.byte6; + rlevel_byte[6].delay = lmc_rlevel_rank.s.byte5; + rlevel_byte[5].delay = lmc_rlevel_rank.s.byte4; + /* ECC */ + rlevel_byte[4].delay = lmc_rlevel_rank.s.byte8; + } else { + rlevel_byte[7].delay = lmc_rlevel_rank.s.byte7; + rlevel_byte[6].delay = lmc_rlevel_rank.s.byte6; + rlevel_byte[5].delay = lmc_rlevel_rank.s.byte5; + rlevel_byte[4].delay = lmc_rlevel_rank.s.byte4; + } + } else { + rlevel_byte[8].delay = lmc_rlevel_rank.s.byte8; /* unused */ + rlevel_byte[7].delay = lmc_rlevel_rank.s.byte7; /* unused */ + rlevel_byte[6].delay = lmc_rlevel_rank.s.byte6; /* unused */ + rlevel_byte[5].delay = lmc_rlevel_rank.s.byte5; /* unused */ + rlevel_byte[4].delay = lmc_rlevel_rank.s.byte4; /* ECC */ + } + + rlevel_byte[3].delay = lmc_rlevel_rank.s.byte3; + rlevel_byte[2].delay = lmc_rlevel_rank.s.byte2; + rlevel_byte[1].delay = lmc_rlevel_rank.s.byte1; + rlevel_byte[0].delay = lmc_rlevel_rank.s.byte0; +} + +static void pack_rlevel_settings(int if_bytemask, int ecc_ena, + struct rlevel_byte_data *rlevel_byte, + union cvmx_lmcx_rlevel_rankx + *final_rlevel_rank) +{ + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank = *final_rlevel_rank; + + if ((if_bytemask & 0xff) == 0xff) { + if (ecc_ena) { + lmc_rlevel_rank.s.byte7 = rlevel_byte[8].delay; + lmc_rlevel_rank.s.byte6 = rlevel_byte[7].delay; + lmc_rlevel_rank.s.byte5 = rlevel_byte[6].delay; + lmc_rlevel_rank.s.byte4 = rlevel_byte[5].delay; + /* ECC */ + lmc_rlevel_rank.s.byte8 = rlevel_byte[4].delay; + } else { + lmc_rlevel_rank.s.byte7 = rlevel_byte[7].delay; + lmc_rlevel_rank.s.byte6 = rlevel_byte[6].delay; + lmc_rlevel_rank.s.byte5 = rlevel_byte[5].delay; + lmc_rlevel_rank.s.byte4 = rlevel_byte[4].delay; + } + } else { + lmc_rlevel_rank.s.byte8 = rlevel_byte[8].delay; + lmc_rlevel_rank.s.byte7 = rlevel_byte[7].delay; + lmc_rlevel_rank.s.byte6 = rlevel_byte[6].delay; + lmc_rlevel_rank.s.byte5 = rlevel_byte[5].delay; + lmc_rlevel_rank.s.byte4 = rlevel_byte[4].delay; + } + + lmc_rlevel_rank.s.byte3 = rlevel_byte[3].delay; + lmc_rlevel_rank.s.byte2 = rlevel_byte[2].delay; + lmc_rlevel_rank.s.byte1 = rlevel_byte[1].delay; + lmc_rlevel_rank.s.byte0 = rlevel_byte[0].delay; + + *final_rlevel_rank = lmc_rlevel_rank; +} + +/////////////////// These are the RLEVEL settings display routines + +// flags +#define WITH_NOTHING 0 +#define WITH_SCORE 1 +#define WITH_AVERAGE 2 +#define WITH_FINAL 4 +#define WITH_COMPUTE 8 + +static void do_display_rl(int if_num, + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank, + int rank, int flags, int score) +{ + char score_buf[16]; + char *msg_buf; + char hex_buf[20]; + + if (flags & WITH_SCORE) { + snprintf(score_buf, sizeof(score_buf), "(%d)", score); + } else { + score_buf[0] = ' '; + score_buf[1] = 0; + } + + if (flags & WITH_AVERAGE) { + msg_buf = " DELAY AVERAGES "; + } else if (flags & WITH_FINAL) { + msg_buf = " FINAL SETTINGS "; + } else if (flags & WITH_COMPUTE) { + msg_buf = " COMPUTED DELAYS "; + } else { + snprintf(hex_buf, sizeof(hex_buf), "0x%016llX", + (unsigned long long)lmc_rlevel_rank.u64); + msg_buf = hex_buf; + } + + debug("N0.LMC%d.R%d: Rlevel Rank %#4x, %s : %5d %5d %5d %5d %5d %5d %5d %5d %5d %s\n", + if_num, rank, lmc_rlevel_rank.s.status, msg_buf, + lmc_rlevel_rank.s.byte8, lmc_rlevel_rank.s.byte7, + lmc_rlevel_rank.s.byte6, lmc_rlevel_rank.s.byte5, + lmc_rlevel_rank.s.byte4, lmc_rlevel_rank.s.byte3, + lmc_rlevel_rank.s.byte2, lmc_rlevel_rank.s.byte1, + lmc_rlevel_rank.s.byte0, score_buf); +} + +static void display_rl(int if_num, + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank, int rank) +{ + do_display_rl(if_num, lmc_rlevel_rank, rank, 0, 0); +} + +static void display_rl_with_score(int if_num, + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank, + int rank, int score) +{ + do_display_rl(if_num, lmc_rlevel_rank, rank, 1, score); +} + +static void display_rl_with_final(int if_num, + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank, + int rank) +{ + do_display_rl(if_num, lmc_rlevel_rank, rank, 4, 0); +} + +static void display_rl_with_computed(int if_num, + union cvmx_lmcx_rlevel_rankx + lmc_rlevel_rank, int rank, int score) +{ + do_display_rl(if_num, lmc_rlevel_rank, rank, 9, score); +} + +// flag values +#define WITH_RODT_BLANK 0 +#define WITH_RODT_SKIPPING 1 +#define WITH_RODT_BESTROW 2 +#define WITH_RODT_BESTSCORE 3 +// control +#define SKIP_SKIPPING 1 + +static const char *with_rodt_canned_msgs[4] = { + " ", "SKIPPING ", "BEST ROW ", "BEST SCORE" +}; + +static void display_rl_with_rodt(int if_num, + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank, + int rank, int score, + int nom_ohms, int rodt_ohms, int flag) +{ + const char *msg_buf; + char set_buf[20]; + +#if SKIP_SKIPPING + if (flag == WITH_RODT_SKIPPING) + return; +#endif + + msg_buf = with_rodt_canned_msgs[flag]; + if (nom_ohms < 0) { + snprintf(set_buf, sizeof(set_buf), " RODT %3d ", + rodt_ohms); + } else { + snprintf(set_buf, sizeof(set_buf), "NOM %3d RODT %3d", nom_ohms, + rodt_ohms); + } + + debug("N0.LMC%d.R%d: Rlevel %s %s : %5d %5d %5d %5d %5d %5d %5d %5d %5d (%d)\n", + if_num, rank, set_buf, msg_buf, lmc_rlevel_rank.s.byte8, + lmc_rlevel_rank.s.byte7, lmc_rlevel_rank.s.byte6, + lmc_rlevel_rank.s.byte5, lmc_rlevel_rank.s.byte4, + lmc_rlevel_rank.s.byte3, lmc_rlevel_rank.s.byte2, + lmc_rlevel_rank.s.byte1, lmc_rlevel_rank.s.byte0, score); +} + +static void do_display_wl(int if_num, + union cvmx_lmcx_wlevel_rankx lmc_wlevel_rank, + int rank, int flags) +{ + char *msg_buf; + char hex_buf[20]; + + if (flags & WITH_FINAL) { + msg_buf = " FINAL SETTINGS "; + } else { + snprintf(hex_buf, sizeof(hex_buf), "0x%016llX", + (unsigned long long)lmc_wlevel_rank.u64); + msg_buf = hex_buf; + } + + debug("N0.LMC%d.R%d: Wlevel Rank %#4x, %s : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n", + if_num, rank, lmc_wlevel_rank.s.status, msg_buf, + lmc_wlevel_rank.s.byte8, lmc_wlevel_rank.s.byte7, + lmc_wlevel_rank.s.byte6, lmc_wlevel_rank.s.byte5, + lmc_wlevel_rank.s.byte4, lmc_wlevel_rank.s.byte3, + lmc_wlevel_rank.s.byte2, lmc_wlevel_rank.s.byte1, + lmc_wlevel_rank.s.byte0); +} + +static void display_wl(int if_num, + union cvmx_lmcx_wlevel_rankx lmc_wlevel_rank, int rank) +{ + do_display_wl(if_num, lmc_wlevel_rank, rank, WITH_NOTHING); +} + +static void display_wl_with_final(int if_num, + union cvmx_lmcx_wlevel_rankx lmc_wlevel_rank, + int rank) +{ + do_display_wl(if_num, lmc_wlevel_rank, rank, WITH_FINAL); +} + +// pretty-print bitmask adjuster +static u64 ppbm(u64 bm) +{ + if (bm != 0ul) { + while ((bm & 0x0fful) == 0ul) + bm >>= 4; + } + + return bm; +} + +// xlate PACKED index to UNPACKED index to use with rlevel_byte +#define XPU(i, e) (((i) < 4) ? (i) : (((i) < 8) ? (i) + (e) : 4)) +// xlate UNPACKED index to PACKED index to use with rlevel_bitmask +#define XUP(i, e) (((i) < 4) ? (i) : (e) ? (((i) > 4) ? (i) - 1 : 8) : (i)) + +// flag values +#define WITH_WL_BITMASKS 0 +#define WITH_RL_BITMASKS 1 +#define WITH_RL_MASK_SCORES 2 +#define WITH_RL_SEQ_SCORES 3 + +static void do_display_bm(int if_num, int rank, void *bm, + int flags, int ecc) +{ + if (flags == WITH_WL_BITMASKS) { + // wlevel_bitmask array in PACKED index order, so just + // print them + int *bitmasks = (int *)bm; + + debug("N0.LMC%d.R%d: Wlevel Debug Bitmasks : %05x %05x %05x %05x %05x %05x %05x %05x %05x\n", + if_num, rank, bitmasks[8], bitmasks[7], bitmasks[6], + bitmasks[5], bitmasks[4], bitmasks[3], bitmasks[2], + bitmasks[1], bitmasks[0] + ); + } else if (flags == WITH_RL_BITMASKS) { + // rlevel_bitmask array in PACKED index order, so just + // print them + struct rlevel_bitmask *rlevel_bitmask = + (struct rlevel_bitmask *)bm; + + debug("N0.LMC%d.R%d: Rlevel Debug Bitmasks 8:0 : %05llx %05llx %05llx %05llx %05llx %05llx %05llx %05llx %05llx\n", + if_num, rank, ppbm(rlevel_bitmask[8].bm), + ppbm(rlevel_bitmask[7].bm), ppbm(rlevel_bitmask[6].bm), + ppbm(rlevel_bitmask[5].bm), ppbm(rlevel_bitmask[4].bm), + ppbm(rlevel_bitmask[3].bm), ppbm(rlevel_bitmask[2].bm), + ppbm(rlevel_bitmask[1].bm), ppbm(rlevel_bitmask[0].bm) + ); + } else if (flags == WITH_RL_MASK_SCORES) { + // rlevel_bitmask array in PACKED index order, so just + // print them + struct rlevel_bitmask *rlevel_bitmask = + (struct rlevel_bitmask *)bm; + + debug("N0.LMC%d.R%d: Rlevel Debug Bitmask Scores 8:0 : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n", + if_num, rank, rlevel_bitmask[8].errs, + rlevel_bitmask[7].errs, rlevel_bitmask[6].errs, + rlevel_bitmask[5].errs, rlevel_bitmask[4].errs, + rlevel_bitmask[3].errs, rlevel_bitmask[2].errs, + rlevel_bitmask[1].errs, rlevel_bitmask[0].errs); + } else if (flags == WITH_RL_SEQ_SCORES) { + // rlevel_byte array in UNPACKED index order, so xlate + // and print them + struct rlevel_byte_data *rlevel_byte = + (struct rlevel_byte_data *)bm; + + debug("N0.LMC%d.R%d: Rlevel Debug Non-seq Scores 8:0 : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n", + if_num, rank, rlevel_byte[XPU(8, ecc)].sqerrs, + rlevel_byte[XPU(7, ecc)].sqerrs, + rlevel_byte[XPU(6, ecc)].sqerrs, + rlevel_byte[XPU(5, ecc)].sqerrs, + rlevel_byte[XPU(4, ecc)].sqerrs, + rlevel_byte[XPU(3, ecc)].sqerrs, + rlevel_byte[XPU(2, ecc)].sqerrs, + rlevel_byte[XPU(1, ecc)].sqerrs, + rlevel_byte[XPU(0, ecc)].sqerrs); + } +} + +static void display_wl_bm(int if_num, int rank, int *bitmasks) +{ + do_display_bm(if_num, rank, (void *)bitmasks, WITH_WL_BITMASKS, 0); +} + +static void display_rl_bm(int if_num, int rank, + struct rlevel_bitmask *bitmasks, int ecc_ena) +{ + do_display_bm(if_num, rank, (void *)bitmasks, WITH_RL_BITMASKS, + ecc_ena); +} + +static void display_rl_bm_scores(int if_num, int rank, + struct rlevel_bitmask *bitmasks, int ecc_ena) +{ + do_display_bm(if_num, rank, (void *)bitmasks, WITH_RL_MASK_SCORES, + ecc_ena); +} + +static void display_rl_seq_scores(int if_num, int rank, + struct rlevel_byte_data *bytes, int ecc_ena) +{ + do_display_bm(if_num, rank, (void *)bytes, WITH_RL_SEQ_SCORES, ecc_ena); +} + +#define RODT_OHMS_COUNT 8 +#define RTT_NOM_OHMS_COUNT 8 +#define RTT_NOM_TABLE_COUNT 8 +#define RTT_WR_OHMS_COUNT 8 +#define DIC_OHMS_COUNT 3 +#define DRIVE_STRENGTH_COUNT 15 + +static unsigned char ddr4_rodt_ohms[RODT_OHMS_COUNT] = { + 0, 40, 60, 80, 120, 240, 34, 48 }; +static unsigned char ddr4_rtt_nom_ohms[RTT_NOM_OHMS_COUNT] = { + 0, 60, 120, 40, 240, 48, 80, 34 }; +static unsigned char ddr4_rtt_nom_table[RTT_NOM_TABLE_COUNT] = { + 0, 4, 2, 6, 1, 5, 3, 7 }; +// setting HiZ ohms to 99 for computed vref +static unsigned char ddr4_rtt_wr_ohms[RTT_WR_OHMS_COUNT] = { + 0, 120, 240, 99, 80 }; +static unsigned char ddr4_dic_ohms[DIC_OHMS_COUNT] = { 34, 48 }; +static short ddr4_drive_strength[DRIVE_STRENGTH_COUNT] = { + 0, 0, 26, 30, 34, 40, 48, 68, 0, 0, 0, 0, 0, 0, 0 }; +static short ddr4_dqx_strength[DRIVE_STRENGTH_COUNT] = { + 0, 24, 27, 30, 34, 40, 48, 60, 0, 0, 0, 0, 0, 0, 0 }; +struct impedence_values ddr4_impedence_val = { + .rodt_ohms = ddr4_rodt_ohms, + .rtt_nom_ohms = ddr4_rtt_nom_ohms, + .rtt_nom_table = ddr4_rtt_nom_table, + .rtt_wr_ohms = ddr4_rtt_wr_ohms, + .dic_ohms = ddr4_dic_ohms, + .drive_strength = ddr4_drive_strength, + .dqx_strength = ddr4_dqx_strength, +}; + +static unsigned char ddr3_rodt_ohms[RODT_OHMS_COUNT] = { + 0, 20, 30, 40, 60, 120, 0, 0 }; +static unsigned char ddr3_rtt_nom_ohms[RTT_NOM_OHMS_COUNT] = { + 0, 60, 120, 40, 20, 30, 0, 0 }; +static unsigned char ddr3_rtt_nom_table[RTT_NOM_TABLE_COUNT] = { + 0, 2, 1, 3, 5, 4, 0, 0 }; +static unsigned char ddr3_rtt_wr_ohms[RTT_WR_OHMS_COUNT] = { 0, 60, 120 }; +static unsigned char ddr3_dic_ohms[DIC_OHMS_COUNT] = { 40, 34 }; +static short ddr3_drive_strength[DRIVE_STRENGTH_COUNT] = { + 0, 24, 27, 30, 34, 40, 48, 60, 0, 0, 0, 0, 0, 0, 0 }; +static struct impedence_values ddr3_impedence_val = { + .rodt_ohms = ddr3_rodt_ohms, + .rtt_nom_ohms = ddr3_rtt_nom_ohms, + .rtt_nom_table = ddr3_rtt_nom_table, + .rtt_wr_ohms = ddr3_rtt_wr_ohms, + .dic_ohms = ddr3_dic_ohms, + .drive_strength = ddr3_drive_strength, + .dqx_strength = ddr3_drive_strength, +}; + +static u64 hertz_to_psecs(u64 hertz) +{ + /* Clock in psecs */ + return divide_nint((u64)1000 * 1000 * 1000 * 1000, hertz); +} + +#define DIVIDEND_SCALE 1000 /* Scale to avoid rounding error. */ + +static u64 psecs_to_mts(u64 psecs) +{ + return divide_nint(divide_nint((u64)(2 * 1000000 * DIVIDEND_SCALE), + psecs), DIVIDEND_SCALE); +} + +#define WITHIN(v, b, m) (((v) >= ((b) - (m))) && ((v) <= ((b) + (m)))) + +static unsigned long pretty_psecs_to_mts(u64 psecs) +{ + u64 ret = 0; // default to error + + if (WITHIN(psecs, 2500, 1)) + ret = 800; + else if (WITHIN(psecs, 1875, 1)) + ret = 1066; + else if (WITHIN(psecs, 1500, 1)) + ret = 1333; + else if (WITHIN(psecs, 1250, 1)) + ret = 1600; + else if (WITHIN(psecs, 1071, 1)) + ret = 1866; + else if (WITHIN(psecs, 937, 1)) + ret = 2133; + else if (WITHIN(psecs, 833, 1)) + ret = 2400; + else if (WITHIN(psecs, 750, 1)) + ret = 2666; + return ret; +} + +static u64 mts_to_hertz(u64 mts) +{ + return ((mts * 1000 * 1000) / 2); +} + +static int compute_rc3x(int64_t tclk_psecs) +{ + long speed; + long tclk_psecs_min, tclk_psecs_max; + long data_rate_mhz, data_rate_mhz_min, data_rate_mhz_max; + int rc3x; + +#define ENCODING_BASE 1240 + + data_rate_mhz = psecs_to_mts(tclk_psecs); + + /* + * 2400 MT/s is a special case. Using integer arithmetic it rounds + * from 833 psecs to 2401 MT/s. Force it to 2400 to pick the + * proper setting from the table. + */ + if (tclk_psecs == 833) + data_rate_mhz = 2400; + + for (speed = ENCODING_BASE; speed < 3200; speed += 20) { + int error = 0; + + /* Clock in psecs */ + tclk_psecs_min = hertz_to_psecs(mts_to_hertz(speed + 00)); + /* Clock in psecs */ + tclk_psecs_max = hertz_to_psecs(mts_to_hertz(speed + 18)); + + data_rate_mhz_min = psecs_to_mts(tclk_psecs_min); + data_rate_mhz_max = psecs_to_mts(tclk_psecs_max); + + /* Force alingment to multiple to avound rounding errors. */ + data_rate_mhz_min = ((data_rate_mhz_min + 18) / 20) * 20; + data_rate_mhz_max = ((data_rate_mhz_max + 18) / 20) * 20; + + error += (speed + 00 != data_rate_mhz_min); + error += (speed + 20 != data_rate_mhz_max); + + rc3x = (speed - ENCODING_BASE) / 20; + + if (data_rate_mhz <= (speed + 20)) + break; + } + + return rc3x; +} + +/* + * static global variables needed, so that functions (loops) can be + * restructured from the main huge function. Its not elegant, but the + * only way to break the original functions like init_octeon3_ddr3_interface() + * into separate logical smaller functions with less indentation levels. + */ +static int if_num __section(".data"); +static u32 if_mask __section(".data"); +static int ddr_hertz __section(".data"); + +static struct ddr_conf *ddr_conf __section(".data"); +static const struct dimm_odt_config *odt_1rank_config __section(".data"); +static const struct dimm_odt_config *odt_2rank_config __section(".data"); +static const struct dimm_odt_config *odt_4rank_config __section(".data"); +static struct dimm_config *dimm_config_table __section(".data"); +static const struct dimm_odt_config *odt_config __section(".data"); +static const struct ddr3_custom_config *c_cfg __section(".data"); + +static int odt_idx __section(".data"); + +static ulong tclk_psecs __section(".data"); +static ulong eclk_psecs __section(".data"); + +static int row_bits __section(".data"); +static int col_bits __section(".data"); +static int num_banks __section(".data"); +static int num_ranks __section(".data"); +static int dram_width __section(".data"); +static int dimm_count __section(".data"); +/* Accumulate and report all the errors before giving up */ +static int fatal_error __section(".data"); +/* Flag that indicates safe DDR settings should be used */ +static int safe_ddr_flag __section(".data"); +/* Octeon II Default: 64bit interface width */ +static int if_64b __section(".data"); +static int if_bytemask __section(".data"); +static u32 mem_size_mbytes __section(".data"); +static unsigned int didx __section(".data"); +static int bank_bits __section(".data"); +static int bunk_enable __section(".data"); +static int rank_mask __section(".data"); +static int column_bits_start __section(".data"); +static int row_lsb __section(".data"); +static int pbank_lsb __section(".data"); +static int use_ecc __section(".data"); +static int mtb_psec __section(".data"); +static short ftb_dividend __section(".data"); +static short ftb_divisor __section(".data"); +static int taamin __section(".data"); +static int tckmin __section(".data"); +static int cl __section(".data"); +static int min_cas_latency __section(".data"); +static int max_cas_latency __section(".data"); +static int override_cas_latency __section(".data"); +static int ddr_rtt_nom_auto __section(".data"); +static int ddr_rodt_ctl_auto __section(".data"); + +static int spd_addr __section(".data"); +static int spd_org __section(".data"); +static int spd_banks __section(".data"); +static int spd_rdimm __section(".data"); +static int spd_dimm_type __section(".data"); +static int spd_ecc __section(".data"); +static u32 spd_cas_latency __section(".data"); +static int spd_mtb_dividend __section(".data"); +static int spd_mtb_divisor __section(".data"); +static int spd_tck_min __section(".data"); +static int spd_taa_min __section(".data"); +static int spd_twr __section(".data"); +static int spd_trcd __section(".data"); +static int spd_trrd __section(".data"); +static int spd_trp __section(".data"); +static int spd_tras __section(".data"); +static int spd_trc __section(".data"); +static int spd_trfc __section(".data"); +static int spd_twtr __section(".data"); +static int spd_trtp __section(".data"); +static int spd_tfaw __section(".data"); +static int spd_addr_mirror __section(".data"); +static int spd_package __section(".data"); +static int spd_rawcard __section(".data"); +static int spd_rawcard_aorb __section(".data"); +static int spd_rdimm_registers __section(".data"); +static int spd_thermal_sensor __section(".data"); + +static int is_stacked_die __section(".data"); +static int is_3ds_dimm __section(".data"); +// 3DS: logical ranks per package rank +static int lranks_per_prank __section(".data"); +// 3DS: logical ranks bits +static int lranks_bits __section(".data"); +// in Mbits; only used for 3DS +static int die_capacity __section(".data"); + +static enum ddr_type ddr_type __section(".data"); + +static int twr __section(".data"); +static int trcd __section(".data"); +static int trrd __section(".data"); +static int trp __section(".data"); +static int tras __section(".data"); +static int trc __section(".data"); +static int trfc __section(".data"); +static int twtr __section(".data"); +static int trtp __section(".data"); +static int tfaw __section(".data"); + +static int ddr4_tckavgmin __section(".data"); +static int ddr4_tckavgmax __section(".data"); +static int ddr4_trdcmin __section(".data"); +static int ddr4_trpmin __section(".data"); +static int ddr4_trasmin __section(".data"); +static int ddr4_trcmin __section(".data"); +static int ddr4_trfc1min __section(".data"); +static int ddr4_trfc2min __section(".data"); +static int ddr4_trfc4min __section(".data"); +static int ddr4_tfawmin __section(".data"); +static int ddr4_trrd_smin __section(".data"); +static int ddr4_trrd_lmin __section(".data"); +static int ddr4_tccd_lmin __section(".data"); + +static int wl_mask_err __section(".data"); +static int wl_loops __section(".data"); +static int default_rtt_nom[4] __section(".data"); +static int dyn_rtt_nom_mask __section(".data"); +static struct impedence_values *imp_val __section(".data"); +static char default_rodt_ctl __section(".data"); +// default to disabled (ie, try LMC restart, not chip reset) +static int ddr_disable_chip_reset __section(".data"); +static const char *dimm_type_name __section(".data"); +static int match_wl_rtt_nom __section(".data"); + +#if SWL_TRY_HWL_ALT +struct hwl_alt_by_rank { + u16 hwl_alt_mask; // mask of bytelanes with alternate + u16 hwl_alt_delay[9]; // bytelane alternate avail if mask=1 +}; + +static struct hwl_alt_by_rank hwl_alts[4] __section(".data"); +#endif /* SWL_TRY_HWL_ALT */ + +#define DEFAULT_INTERNAL_VREF_TRAINING_LIMIT 3 // was: 5 +static int internal_retries __section(".data"); + +static int deskew_training_errors __section(".data"); +static struct deskew_counts deskew_training_results __section(".data"); +static int disable_deskew_training __section(".data"); +static int restart_if_dsk_incomplete __section(".data"); +static int dac_eval_retries __section(".data"); +static int dac_settings[9] __section(".data"); +static int num_samples __section(".data"); +static int sample __section(".data"); +static int lane __section(".data"); +static int last_lane __section(".data"); +static int total_dac_eval_retries __section(".data"); +static int dac_eval_exhausted __section(".data"); + +#define DEFAULT_DAC_SAMPLES 7 // originally was 5 +#define DAC_RETRIES_LIMIT 2 + +struct bytelane_sample { + s16 bytes[DEFAULT_DAC_SAMPLES]; +}; + +static struct bytelane_sample lanes[9] __section(".data"); + +static char disable_sequential_delay_check __section(".data"); +static int wl_print __section(".data"); + +#if ALLOW_BY_RANK_INIT +static int enable_by_rank_init __section(".data"); +static int saved_rank_mask __section(".data"); +static int by_rank __section(".data"); +static struct deskew_data rank_dsk[4] __section(".data"); +static struct dac_data rank_dac[4] __section(".data"); +#endif /* ALLOW_BY_RANK_INIT */ + +// todo: perhaps remove node at some time completely? +static int node __section(".data"); +static int base_cl __section(".data"); + +/* Parameters from DDR3 Specifications */ +#define DDR3_TREFI 7800000 /* 7.8 us */ +#define DDR3_ZQCS 80000ull /* 80 ns */ +#define DDR3_ZQCS_INTERNAL 1280000000ull /* 128ms/100 */ +#define DDR3_TCKE 5000 /* 5 ns */ +#define DDR3_TMRD 4 /* 4 nCK */ +#define DDR3_TDLLK 512 /* 512 nCK */ +#define DDR3_TMPRR 1 /* 1 nCK */ +#define DDR3_TWLMRD 40 /* 40 nCK */ +#define DDR3_TWLDQSEN 25 /* 25 nCK */ + +/* Parameters from DDR4 Specifications */ +#define DDR4_TMRD 8 /* 8 nCK */ +#define DDR4_TDLLK 768 /* 768 nCK */ + +static void lmc_config(struct ddr_priv *priv) +{ + union cvmx_lmcx_config cfg; + char *s; + + cfg.u64 = 0; + + cfg.cn78xx.ecc_ena = use_ecc; + cfg.cn78xx.row_lsb = encode_row_lsb_ddr3(row_lsb); + cfg.cn78xx.pbank_lsb = encode_pbank_lsb_ddr3(pbank_lsb); + + cfg.cn78xx.idlepower = 0; /* Disabled */ + + s = lookup_env(priv, "ddr_idlepower"); + if (s) + cfg.cn78xx.idlepower = simple_strtoul(s, NULL, 0); + + cfg.cn78xx.forcewrite = 0; /* Disabled */ + /* Include memory reference address in the ECC */ + cfg.cn78xx.ecc_adr = 1; + + s = lookup_env(priv, "ddr_ecc_adr"); + if (s) + cfg.cn78xx.ecc_adr = simple_strtoul(s, NULL, 0); + + cfg.cn78xx.reset = 0; + + /* + * Program LMC0_CONFIG[24:18], ref_zqcs_int(6:0) to + * RND-DN(tREFI/clkPeriod/512) Program LMC0_CONFIG[36:25], + * ref_zqcs_int(18:7) to + * RND-DN(ZQCS_Interval/clkPeriod/(512*128)). Note that this + * value should always be greater than 32, to account for + * resistor calibration delays. + */ + + cfg.cn78xx.ref_zqcs_int = ((DDR3_TREFI / tclk_psecs / 512) & 0x7f); + cfg.cn78xx.ref_zqcs_int |= + ((max(33ull, (DDR3_ZQCS_INTERNAL / (tclk_psecs / 100) / + (512 * 128))) & 0xfff) << 7); + + cfg.cn78xx.early_dqx = 1; /* Default to enabled */ + + s = lookup_env(priv, "ddr_early_dqx"); + if (!s) + s = lookup_env(priv, "ddr%d_early_dqx", if_num); + + if (s) + cfg.cn78xx.early_dqx = simple_strtoul(s, NULL, 0); + + cfg.cn78xx.sref_with_dll = 0; + + cfg.cn78xx.rank_ena = bunk_enable; + cfg.cn78xx.rankmask = rank_mask; /* Set later */ + cfg.cn78xx.mirrmask = (spd_addr_mirror << 1 | spd_addr_mirror << 3) & + rank_mask; + /* Set once and don't change it. */ + cfg.cn78xx.init_status = rank_mask; + cfg.cn78xx.early_unload_d0_r0 = 0; + cfg.cn78xx.early_unload_d0_r1 = 0; + cfg.cn78xx.early_unload_d1_r0 = 0; + cfg.cn78xx.early_unload_d1_r1 = 0; + cfg.cn78xx.scrz = 0; + if (octeon_is_cpuid(OCTEON_CN70XX)) + cfg.cn78xx.mode32b = 1; /* Read-only. Always 1. */ + cfg.cn78xx.mode_x4dev = (dram_width == 4) ? 1 : 0; + cfg.cn78xx.bg2_enable = ((ddr_type == DDR4_DRAM) && + (dram_width == 16)) ? 0 : 1; + + s = lookup_env_ull(priv, "ddr_config"); + if (s) + cfg.u64 = simple_strtoull(s, NULL, 0); + debug("LMC_CONFIG : 0x%016llx\n", + cfg.u64); + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), cfg.u64); +} + +static void lmc_control(struct ddr_priv *priv) +{ + union cvmx_lmcx_control ctrl; + char *s; + + ctrl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + ctrl.s.rdimm_ena = spd_rdimm; + ctrl.s.bwcnt = 0; /* Clear counter later */ + if (spd_rdimm) + ctrl.s.ddr2t = (safe_ddr_flag ? 1 : c_cfg->ddr2t_rdimm); + else + ctrl.s.ddr2t = (safe_ddr_flag ? 1 : c_cfg->ddr2t_udimm); + ctrl.s.pocas = 0; + ctrl.s.fprch2 = (safe_ddr_flag ? 2 : c_cfg->fprch2); + ctrl.s.throttle_rd = safe_ddr_flag ? 1 : 0; + ctrl.s.throttle_wr = safe_ddr_flag ? 1 : 0; + ctrl.s.inorder_rd = safe_ddr_flag ? 1 : 0; + ctrl.s.inorder_wr = safe_ddr_flag ? 1 : 0; + ctrl.s.elev_prio_dis = safe_ddr_flag ? 1 : 0; + /* discards writes to addresses that don't exist in the DRAM */ + ctrl.s.nxm_write_en = 0; + ctrl.s.max_write_batch = 8; + ctrl.s.xor_bank = 1; + ctrl.s.auto_dclkdis = 1; + ctrl.s.int_zqcs_dis = 0; + ctrl.s.ext_zqcs_dis = 0; + ctrl.s.bprch = 1; + ctrl.s.wodt_bprch = 1; + ctrl.s.rodt_bprch = 1; + + s = lookup_env(priv, "ddr_xor_bank"); + if (s) + ctrl.s.xor_bank = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_2t"); + if (s) + ctrl.s.ddr2t = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_fprch2"); + if (s) + ctrl.s.fprch2 = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_bprch"); + if (s) + ctrl.s.bprch = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_wodt_bprch"); + if (s) + ctrl.s.wodt_bprch = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rodt_bprch"); + if (s) + ctrl.s.rodt_bprch = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_int_zqcs_dis"); + if (s) + ctrl.s.int_zqcs_dis = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_ext_zqcs_dis"); + if (s) + ctrl.s.ext_zqcs_dis = simple_strtoul(s, NULL, 0); + + s = lookup_env_ull(priv, "ddr_control"); + if (s) + ctrl.u64 = simple_strtoull(s, NULL, 0); + + debug("LMC_CONTROL : 0x%016llx\n", + ctrl.u64); + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctrl.u64); +} + +static void lmc_timing_params0(struct ddr_priv *priv) +{ + union cvmx_lmcx_timing_params0 tp0; + unsigned int trp_value; + char *s; + + tp0.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS0(if_num)); + + trp_value = divide_roundup(trp, tclk_psecs) - 1; + debug("TIMING_PARAMS0[TRP]: NEW 0x%x, OLD 0x%x\n", trp_value, + trp_value + + (unsigned int)(divide_roundup(max(4ull * tclk_psecs, 7500ull), + tclk_psecs)) - 4); + s = lookup_env_ull(priv, "ddr_use_old_trp"); + if (s) { + if (!!simple_strtoull(s, NULL, 0)) { + trp_value += + divide_roundup(max(4ull * tclk_psecs, 7500ull), + tclk_psecs) - 4; + debug("TIMING_PARAMS0[trp]: USING OLD 0x%x\n", + trp_value); + } + } + + tp0.cn78xx.txpr = + divide_roundup(max(5ull * tclk_psecs, trfc + 10000ull), + 16 * tclk_psecs); + tp0.cn78xx.trp = trp_value & 0x1f; + tp0.cn78xx.tcksre = + divide_roundup(max(5ull * tclk_psecs, 10000ull), tclk_psecs) - 1; + + if (ddr_type == DDR4_DRAM) { + int tzqinit = 4; // Default to 4, for all DDR4 speed bins + + s = lookup_env(priv, "ddr_tzqinit"); + if (s) + tzqinit = simple_strtoul(s, NULL, 0); + + tp0.cn78xx.tzqinit = tzqinit; + /* Always 8. */ + tp0.cn78xx.tzqcs = divide_roundup(128 * tclk_psecs, + (16 * tclk_psecs)); + tp0.cn78xx.tcke = + divide_roundup(max(3 * tclk_psecs, (ulong)DDR3_TCKE), + tclk_psecs) - 1; + tp0.cn78xx.tmrd = + divide_roundup((DDR4_TMRD * tclk_psecs), tclk_psecs) - 1; + tp0.cn78xx.tmod = 25; /* 25 is the max allowed */ + tp0.cn78xx.tdllk = divide_roundup(DDR4_TDLLK, 256); + } else { + tp0.cn78xx.tzqinit = + divide_roundup(max(512ull * tclk_psecs, 640000ull), + (256 * tclk_psecs)); + tp0.cn78xx.tzqcs = + divide_roundup(max(64ull * tclk_psecs, DDR3_ZQCS), + (16 * tclk_psecs)); + tp0.cn78xx.tcke = divide_roundup(DDR3_TCKE, tclk_psecs) - 1; + tp0.cn78xx.tmrd = + divide_roundup((DDR3_TMRD * tclk_psecs), tclk_psecs) - 1; + tp0.cn78xx.tmod = + divide_roundup(max(12ull * tclk_psecs, 15000ull), + tclk_psecs) - 1; + tp0.cn78xx.tdllk = divide_roundup(DDR3_TDLLK, 256); + } + + s = lookup_env_ull(priv, "ddr_timing_params0"); + if (s) + tp0.u64 = simple_strtoull(s, NULL, 0); + debug("TIMING_PARAMS0 : 0x%016llx\n", + tp0.u64); + lmc_wr(priv, CVMX_LMCX_TIMING_PARAMS0(if_num), tp0.u64); +} + +static void lmc_timing_params1(struct ddr_priv *priv) +{ + union cvmx_lmcx_timing_params1 tp1; + unsigned int txp, temp_trcd, trfc_dlr; + char *s; + + tp1.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS1(if_num)); + + /* .cn70xx. */ + tp1.s.tmprr = divide_roundup(DDR3_TMPRR * tclk_psecs, tclk_psecs) - 1; + + tp1.cn78xx.tras = divide_roundup(tras, tclk_psecs) - 1; + + temp_trcd = divide_roundup(trcd, tclk_psecs); + if (temp_trcd > 15) { + debug("TIMING_PARAMS1[trcd]: need extension bit for 0x%x\n", + temp_trcd); + } + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && temp_trcd > 15) { + /* + * Let .trcd=0 serve as a flag that the field has + * overflowed. Must use Additive Latency mode as a + * workaround. + */ + temp_trcd = 0; + } + tp1.cn78xx.trcd = (temp_trcd >> 0) & 0xf; + tp1.cn78xx.trcd_ext = (temp_trcd >> 4) & 0x1; + + tp1.cn78xx.twtr = divide_roundup(twtr, tclk_psecs) - 1; + tp1.cn78xx.trfc = divide_roundup(trfc, 8 * tclk_psecs); + + if (ddr_type == DDR4_DRAM) { + /* Workaround bug 24006. Use Trrd_l. */ + tp1.cn78xx.trrd = + divide_roundup(ddr4_trrd_lmin, tclk_psecs) - 2; + } else { + tp1.cn78xx.trrd = divide_roundup(trrd, tclk_psecs) - 2; + } + + /* + * tXP = max( 3nCK, 7.5 ns) DDR3-800 tCLK = 2500 psec + * tXP = max( 3nCK, 7.5 ns) DDR3-1066 tCLK = 1875 psec + * tXP = max( 3nCK, 6.0 ns) DDR3-1333 tCLK = 1500 psec + * tXP = max( 3nCK, 6.0 ns) DDR3-1600 tCLK = 1250 psec + * tXP = max( 3nCK, 6.0 ns) DDR3-1866 tCLK = 1071 psec + * tXP = max( 3nCK, 6.0 ns) DDR3-2133 tCLK = 937 psec + */ + txp = (tclk_psecs < 1875) ? 6000 : 7500; + txp = divide_roundup(max((unsigned int)(3 * tclk_psecs), txp), + tclk_psecs) - 1; + if (txp > 7) { + debug("TIMING_PARAMS1[txp]: need extension bit for 0x%x\n", + txp); + } + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && txp > 7) + txp = 7; // max it out + tp1.cn78xx.txp = (txp >> 0) & 7; + tp1.cn78xx.txp_ext = (txp >> 3) & 1; + + tp1.cn78xx.twlmrd = divide_roundup(DDR3_TWLMRD * tclk_psecs, + 4 * tclk_psecs); + tp1.cn78xx.twldqsen = divide_roundup(DDR3_TWLDQSEN * tclk_psecs, + 4 * tclk_psecs); + tp1.cn78xx.tfaw = divide_roundup(tfaw, 4 * tclk_psecs); + tp1.cn78xx.txpdll = divide_roundup(max(10ull * tclk_psecs, 24000ull), + tclk_psecs) - 1; + + if (ddr_type == DDR4_DRAM && is_3ds_dimm) { + /* + * 4 Gb: tRFC_DLR = 90 ns + * 8 Gb: tRFC_DLR = 120 ns + * 16 Gb: tRFC_DLR = 190 ns FIXME? + */ + if (die_capacity == 0x1000) // 4 Gbit + trfc_dlr = 90; + else if (die_capacity == 0x2000) // 8 Gbit + trfc_dlr = 120; + else if (die_capacity == 0x4000) // 16 Gbit + trfc_dlr = 190; + else + trfc_dlr = 0; + + if (trfc_dlr == 0) { + debug("N%d.LMC%d: ERROR: tRFC_DLR: die_capacity %u Mbit is illegal\n", + node, if_num, die_capacity); + } else { + tp1.cn78xx.trfc_dlr = + divide_roundup(trfc_dlr * 1000UL, 8 * tclk_psecs); + debug("N%d.LMC%d: TIMING_PARAMS1[trfc_dlr] set to %u\n", + node, if_num, tp1.cn78xx.trfc_dlr); + } + } + + s = lookup_env_ull(priv, "ddr_timing_params1"); + if (s) + tp1.u64 = simple_strtoull(s, NULL, 0); + + debug("TIMING_PARAMS1 : 0x%016llx\n", + tp1.u64); + lmc_wr(priv, CVMX_LMCX_TIMING_PARAMS1(if_num), tp1.u64); +} + +static void lmc_timing_params2(struct ddr_priv *priv) +{ + if (ddr_type == DDR4_DRAM) { + union cvmx_lmcx_timing_params1 tp1; + union cvmx_lmcx_timing_params2 tp2; + int temp_trrd_l; + + tp1.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS1(if_num)); + tp2.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS2(if_num)); + debug("TIMING_PARAMS2 : 0x%016llx\n", + tp2.u64); + + temp_trrd_l = divide_roundup(ddr4_trrd_lmin, tclk_psecs) - 2; + if (temp_trrd_l > 7) + debug("TIMING_PARAMS2[trrd_l]: need extension bit for 0x%x\n", + temp_trrd_l); + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && temp_trrd_l > 7) + temp_trrd_l = 7; // max it out + tp2.cn78xx.trrd_l = (temp_trrd_l >> 0) & 7; + tp2.cn78xx.trrd_l_ext = (temp_trrd_l >> 3) & 1; + + // correct for 1600-2400 + tp2.s.twtr_l = divide_nint(max(4ull * tclk_psecs, 7500ull), + tclk_psecs) - 1; + tp2.s.t_rw_op_max = 7; + tp2.s.trtp = divide_roundup(max(4ull * tclk_psecs, 7500ull), + tclk_psecs) - 1; + + debug("TIMING_PARAMS2 : 0x%016llx\n", + tp2.u64); + lmc_wr(priv, CVMX_LMCX_TIMING_PARAMS2(if_num), tp2.u64); + + /* + * Workaround Errata 25823 - LMC: Possible DDR4 tWTR_L not met + * for Write-to-Read operations to the same Bank Group + */ + if (tp1.cn78xx.twtr < (tp2.s.twtr_l - 4)) { + tp1.cn78xx.twtr = tp2.s.twtr_l - 4; + debug("ERRATA 25823: NEW: TWTR: %d, TWTR_L: %d\n", + tp1.cn78xx.twtr, tp2.s.twtr_l); + debug("TIMING_PARAMS1 : 0x%016llx\n", + tp1.u64); + lmc_wr(priv, CVMX_LMCX_TIMING_PARAMS1(if_num), tp1.u64); + } + } +} + +static void lmc_modereg_params0(struct ddr_priv *priv) +{ + union cvmx_lmcx_modereg_params0 mp0; + int param; + char *s; + + mp0.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num)); + + if (ddr_type == DDR4_DRAM) { + mp0.s.cwl = 0; /* 1600 (1250ps) */ + if (tclk_psecs < 1250) + mp0.s.cwl = 1; /* 1866 (1072ps) */ + if (tclk_psecs < 1072) + mp0.s.cwl = 2; /* 2133 (938ps) */ + if (tclk_psecs < 938) + mp0.s.cwl = 3; /* 2400 (833ps) */ + if (tclk_psecs < 833) + mp0.s.cwl = 4; /* 2666 (750ps) */ + if (tclk_psecs < 750) + mp0.s.cwl = 5; /* 3200 (625ps) */ + } else { + /* + ** CSR CWL CAS write Latency + ** === === ================================= + ** 0 5 ( tCK(avg) >= 2.5 ns) + ** 1 6 (2.5 ns > tCK(avg) >= 1.875 ns) + ** 2 7 (1.875 ns > tCK(avg) >= 1.5 ns) + ** 3 8 (1.5 ns > tCK(avg) >= 1.25 ns) + ** 4 9 (1.25 ns > tCK(avg) >= 1.07 ns) + ** 5 10 (1.07 ns > tCK(avg) >= 0.935 ns) + ** 6 11 (0.935 ns > tCK(avg) >= 0.833 ns) + ** 7 12 (0.833 ns > tCK(avg) >= 0.75 ns) + */ + + mp0.s.cwl = 0; + if (tclk_psecs < 2500) + mp0.s.cwl = 1; + if (tclk_psecs < 1875) + mp0.s.cwl = 2; + if (tclk_psecs < 1500) + mp0.s.cwl = 3; + if (tclk_psecs < 1250) + mp0.s.cwl = 4; + if (tclk_psecs < 1070) + mp0.s.cwl = 5; + if (tclk_psecs < 935) + mp0.s.cwl = 6; + if (tclk_psecs < 833) + mp0.s.cwl = 7; + } + + s = lookup_env(priv, "ddr_cwl"); + if (s) + mp0.s.cwl = simple_strtoul(s, NULL, 0) - 5; + + if (ddr_type == DDR4_DRAM) { + debug("%-45s : %d, [0x%x]\n", "CAS Write Latency CWL, [CSR]", + mp0.s.cwl + 9 + + ((mp0.s.cwl > 2) ? (mp0.s.cwl - 3) * 2 : 0), mp0.s.cwl); + } else { + debug("%-45s : %d, [0x%x]\n", "CAS Write Latency CWL, [CSR]", + mp0.s.cwl + 5, mp0.s.cwl); + } + + mp0.s.mprloc = 0; + mp0.s.mpr = 0; + mp0.s.dll = (ddr_type == DDR4_DRAM); /* 0 for DDR3 and 1 for DDR4 */ + mp0.s.al = 0; + mp0.s.wlev = 0; /* Read Only */ + if (octeon_is_cpuid(OCTEON_CN70XX) || ddr_type == DDR4_DRAM) + mp0.s.tdqs = 0; + else + mp0.s.tdqs = 1; + mp0.s.qoff = 0; + + s = lookup_env(priv, "ddr_cl"); + if (s) { + cl = simple_strtoul(s, NULL, 0); + debug("CAS Latency : %6d\n", + cl); + } + + if (ddr_type == DDR4_DRAM) { + mp0.s.cl = 0x0; + if (cl > 9) + mp0.s.cl = 0x1; + if (cl > 10) + mp0.s.cl = 0x2; + if (cl > 11) + mp0.s.cl = 0x3; + if (cl > 12) + mp0.s.cl = 0x4; + if (cl > 13) + mp0.s.cl = 0x5; + if (cl > 14) + mp0.s.cl = 0x6; + if (cl > 15) + mp0.s.cl = 0x7; + if (cl > 16) + mp0.s.cl = 0x8; + if (cl > 18) + mp0.s.cl = 0x9; + if (cl > 20) + mp0.s.cl = 0xA; + if (cl > 24) + mp0.s.cl = 0xB; + } else { + mp0.s.cl = 0x2; + if (cl > 5) + mp0.s.cl = 0x4; + if (cl > 6) + mp0.s.cl = 0x6; + if (cl > 7) + mp0.s.cl = 0x8; + if (cl > 8) + mp0.s.cl = 0xA; + if (cl > 9) + mp0.s.cl = 0xC; + if (cl > 10) + mp0.s.cl = 0xE; + if (cl > 11) + mp0.s.cl = 0x1; + if (cl > 12) + mp0.s.cl = 0x3; + if (cl > 13) + mp0.s.cl = 0x5; + if (cl > 14) + mp0.s.cl = 0x7; + if (cl > 15) + mp0.s.cl = 0x9; + } + + mp0.s.rbt = 0; /* Read Only. */ + mp0.s.tm = 0; + mp0.s.dllr = 0; + + param = divide_roundup(twr, tclk_psecs); + + if (ddr_type == DDR4_DRAM) { /* DDR4 */ + mp0.s.wrp = 1; + if (param > 12) + mp0.s.wrp = 2; + if (param > 14) + mp0.s.wrp = 3; + if (param > 16) + mp0.s.wrp = 4; + if (param > 18) + mp0.s.wrp = 5; + if (param > 20) + mp0.s.wrp = 6; + if (param > 24) /* RESERVED in DDR4 spec */ + mp0.s.wrp = 7; + } else { /* DDR3 */ + mp0.s.wrp = 1; + if (param > 5) + mp0.s.wrp = 2; + if (param > 6) + mp0.s.wrp = 3; + if (param > 7) + mp0.s.wrp = 4; + if (param > 8) + mp0.s.wrp = 5; + if (param > 10) + mp0.s.wrp = 6; + if (param > 12) + mp0.s.wrp = 7; + } + + mp0.s.ppd = 0; + + s = lookup_env(priv, "ddr_wrp"); + if (s) + mp0.s.wrp = simple_strtoul(s, NULL, 0); + + debug("%-45s : %d, [0x%x]\n", + "Write recovery for auto precharge WRP, [CSR]", param, mp0.s.wrp); + + s = lookup_env_ull(priv, "ddr_modereg_params0"); + if (s) + mp0.u64 = simple_strtoull(s, NULL, 0); + + debug("MODEREG_PARAMS0 : 0x%016llx\n", + mp0.u64); + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num), mp0.u64); +} + +static void lmc_modereg_params1(struct ddr_priv *priv) +{ + union cvmx_lmcx_modereg_params1 mp1; + char *s; + int i; + + mp1.u64 = odt_config[odt_idx].modereg_params1.u64; + +#ifdef CAVIUM_ONLY + /* + * Special request: mismatched DIMM support. Slot 0: 2-Rank, + * Slot 1: 1-Rank + */ + if (rank_mask == 0x7) { /* 2-Rank, 1-Rank */ + mp1.s.rtt_nom_00 = 0; + mp1.s.rtt_nom_01 = 3; /* rttnom_40ohm */ + mp1.s.rtt_nom_10 = 3; /* rttnom_40ohm */ + mp1.s.rtt_nom_11 = 0; + dyn_rtt_nom_mask = 0x6; + } +#endif /* CAVIUM_ONLY */ + + s = lookup_env(priv, "ddr_rtt_nom_mask"); + if (s) + dyn_rtt_nom_mask = simple_strtoul(s, NULL, 0); + + /* + * Save the original rtt_nom settings before sweeping through + * settings. + */ + default_rtt_nom[0] = mp1.s.rtt_nom_00; + default_rtt_nom[1] = mp1.s.rtt_nom_01; + default_rtt_nom[2] = mp1.s.rtt_nom_10; + default_rtt_nom[3] = mp1.s.rtt_nom_11; + + ddr_rtt_nom_auto = c_cfg->ddr_rtt_nom_auto; + + for (i = 0; i < 4; ++i) { + u64 value; + + s = lookup_env(priv, "ddr_rtt_nom_%1d%1d", !!(i & 2), + !!(i & 1)); + if (!s) + s = lookup_env(priv, "ddr%d_rtt_nom_%1d%1d", if_num, + !!(i & 2), !!(i & 1)); + if (s) { + value = simple_strtoul(s, NULL, 0); + mp1.u64 &= ~((u64)0x7 << (i * 12 + 9)); + mp1.u64 |= ((value & 0x7) << (i * 12 + 9)); + default_rtt_nom[i] = value; + ddr_rtt_nom_auto = 0; + } + } + + s = lookup_env(priv, "ddr_rtt_nom"); + if (!s) + s = lookup_env(priv, "ddr%d_rtt_nom", if_num); + if (s) { + u64 value; + + value = simple_strtoul(s, NULL, 0); + + if (dyn_rtt_nom_mask & 1) { + default_rtt_nom[0] = value; + mp1.s.rtt_nom_00 = value; + } + if (dyn_rtt_nom_mask & 2) { + default_rtt_nom[1] = value; + mp1.s.rtt_nom_01 = value; + } + if (dyn_rtt_nom_mask & 4) { + default_rtt_nom[2] = value; + mp1.s.rtt_nom_10 = value; + } + if (dyn_rtt_nom_mask & 8) { + default_rtt_nom[3] = value; + mp1.s.rtt_nom_11 = value; + } + + ddr_rtt_nom_auto = 0; + } + + for (i = 0; i < 4; ++i) { + u64 value; + + s = lookup_env(priv, "ddr_rtt_wr_%1d%1d", !!(i & 2), !!(i & 1)); + if (!s) + s = lookup_env(priv, "ddr%d_rtt_wr_%1d%1d", if_num, + !!(i & 2), !!(i & 1)); + if (s) { + value = simple_strtoul(s, NULL, 0); + insrt_wr(&mp1.u64, i, value); + } + } + + // Make sure 78XX pass 1 has valid RTT_WR settings, because + // configuration files may be set-up for later chips, and + // 78XX pass 1 supports no RTT_WR extension bits + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) { + for (i = 0; i < 4; ++i) { + // if 80 or undefined + if (extr_wr(mp1.u64, i) > 3) { + // FIXME? always insert 120 + insrt_wr(&mp1.u64, i, 1); + debug("RTT_WR_%d%d set to 120 for CN78XX pass 1\n", + !!(i & 2), i & 1); + } + } + } + + s = lookup_env(priv, "ddr_dic"); + if (s) { + u64 value = simple_strtoul(s, NULL, 0); + + for (i = 0; i < 4; ++i) { + mp1.u64 &= ~((u64)0x3 << (i * 12 + 7)); + mp1.u64 |= ((value & 0x3) << (i * 12 + 7)); + } + } + + for (i = 0; i < 4; ++i) { + u64 value; + + s = lookup_env(priv, "ddr_dic_%1d%1d", !!(i & 2), !!(i & 1)); + if (s) { + value = simple_strtoul(s, NULL, 0); + mp1.u64 &= ~((u64)0x3 << (i * 12 + 7)); + mp1.u64 |= ((value & 0x3) << (i * 12 + 7)); + } + } + + s = lookup_env_ull(priv, "ddr_modereg_params1"); + if (s) + mp1.u64 = simple_strtoull(s, NULL, 0); + + debug("RTT_NOM %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_11], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_10], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_01], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_00], + mp1.s.rtt_nom_11, + mp1.s.rtt_nom_10, mp1.s.rtt_nom_01, mp1.s.rtt_nom_00); + + debug("RTT_WR %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 3)], + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 2)], + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 1)], + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 0)], + extr_wr(mp1.u64, 3), + extr_wr(mp1.u64, 2), extr_wr(mp1.u64, 1), extr_wr(mp1.u64, 0)); + + debug("DIC %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->dic_ohms[mp1.s.dic_11], + imp_val->dic_ohms[mp1.s.dic_10], + imp_val->dic_ohms[mp1.s.dic_01], + imp_val->dic_ohms[mp1.s.dic_00], + mp1.s.dic_11, mp1.s.dic_10, mp1.s.dic_01, mp1.s.dic_00); + + debug("MODEREG_PARAMS1 : 0x%016llx\n", + mp1.u64); + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num), mp1.u64); +} + +static void lmc_modereg_params2(struct ddr_priv *priv) +{ + char *s; + int i; + + if (ddr_type == DDR4_DRAM) { + union cvmx_lmcx_modereg_params2 mp2; + + mp2.u64 = odt_config[odt_idx].modereg_params2.u64; + + s = lookup_env(priv, "ddr_rtt_park"); + if (s) { + u64 value = simple_strtoul(s, NULL, 0); + + for (i = 0; i < 4; ++i) { + mp2.u64 &= ~((u64)0x7 << (i * 10 + 0)); + mp2.u64 |= ((value & 0x7) << (i * 10 + 0)); + } + } + + for (i = 0; i < 4; ++i) { + u64 value; + + s = lookup_env(priv, "ddr_rtt_park_%1d%1d", !!(i & 2), + !!(i & 1)); + if (s) { + value = simple_strtoul(s, NULL, 0); + mp2.u64 &= ~((u64)0x7 << (i * 10 + 0)); + mp2.u64 |= ((value & 0x7) << (i * 10 + 0)); + } + } + + s = lookup_env_ull(priv, "ddr_modereg_params2"); + if (s) + mp2.u64 = simple_strtoull(s, NULL, 0); + + debug("RTT_PARK %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_nom_ohms[mp2.s.rtt_park_11], + imp_val->rtt_nom_ohms[mp2.s.rtt_park_10], + imp_val->rtt_nom_ohms[mp2.s.rtt_park_01], + imp_val->rtt_nom_ohms[mp2.s.rtt_park_00], + mp2.s.rtt_park_11, mp2.s.rtt_park_10, mp2.s.rtt_park_01, + mp2.s.rtt_park_00); + + debug("%-45s : 0x%x,0x%x,0x%x,0x%x\n", "VREF_RANGE", + mp2.s.vref_range_11, + mp2.s.vref_range_10, + mp2.s.vref_range_01, mp2.s.vref_range_00); + + debug("%-45s : 0x%x,0x%x,0x%x,0x%x\n", "VREF_VALUE", + mp2.s.vref_value_11, + mp2.s.vref_value_10, + mp2.s.vref_value_01, mp2.s.vref_value_00); + + debug("MODEREG_PARAMS2 : 0x%016llx\n", + mp2.u64); + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS2(if_num), mp2.u64); + } +} + +static void lmc_modereg_params3(struct ddr_priv *priv) +{ + char *s; + + if (ddr_type == DDR4_DRAM) { + union cvmx_lmcx_modereg_params3 mp3; + + mp3.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS3(if_num)); + /* Disable as workaround to Errata 20547 */ + mp3.s.rd_dbi = 0; + mp3.s.tccd_l = max(divide_roundup(ddr4_tccd_lmin, tclk_psecs), + 5ull) - 4; + + s = lookup_env(priv, "ddr_rd_preamble"); + if (s) + mp3.s.rd_preamble = !!simple_strtoul(s, NULL, 0); + + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) { + int delay = 0; + + if (lranks_per_prank == 4 && ddr_hertz >= 1000000000) + delay = 1; + + mp3.s.xrank_add_tccd_l = delay; + mp3.s.xrank_add_tccd_s = delay; + } + + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS3(if_num), mp3.u64); + debug("MODEREG_PARAMS3 : 0x%016llx\n", + mp3.u64); + } +} + +static void lmc_nxm(struct ddr_priv *priv) +{ + union cvmx_lmcx_nxm lmc_nxm; + int num_bits = row_lsb + row_bits + lranks_bits - 26; + char *s; + + lmc_nxm.u64 = lmc_rd(priv, CVMX_LMCX_NXM(if_num)); + + /* .cn78xx. */ + if (rank_mask & 0x1) + lmc_nxm.cn78xx.mem_msb_d0_r0 = num_bits; + if (rank_mask & 0x2) + lmc_nxm.cn78xx.mem_msb_d0_r1 = num_bits; + if (rank_mask & 0x4) + lmc_nxm.cn78xx.mem_msb_d1_r0 = num_bits; + if (rank_mask & 0x8) + lmc_nxm.cn78xx.mem_msb_d1_r1 = num_bits; + + /* Set the mask for non-existent ranks. */ + lmc_nxm.cn78xx.cs_mask = ~rank_mask & 0xff; + + s = lookup_env_ull(priv, "ddr_nxm"); + if (s) + lmc_nxm.u64 = simple_strtoull(s, NULL, 0); + + debug("LMC_NXM : 0x%016llx\n", + lmc_nxm.u64); + lmc_wr(priv, CVMX_LMCX_NXM(if_num), lmc_nxm.u64); +} + +static void lmc_wodt_mask(struct ddr_priv *priv) +{ + union cvmx_lmcx_wodt_mask wodt_mask; + char *s; + + wodt_mask.u64 = odt_config[odt_idx].odt_mask; + + s = lookup_env_ull(priv, "ddr_wodt_mask"); + if (s) + wodt_mask.u64 = simple_strtoull(s, NULL, 0); + + debug("WODT_MASK : 0x%016llx\n", + wodt_mask.u64); + lmc_wr(priv, CVMX_LMCX_WODT_MASK(if_num), wodt_mask.u64); +} + +static void lmc_rodt_mask(struct ddr_priv *priv) +{ + union cvmx_lmcx_rodt_mask rodt_mask; + int rankx; + char *s; + + rodt_mask.u64 = odt_config[odt_idx].rodt_ctl; + + s = lookup_env_ull(priv, "ddr_rodt_mask"); + if (s) + rodt_mask.u64 = simple_strtoull(s, NULL, 0); + + debug("%-45s : 0x%016llx\n", "RODT_MASK", rodt_mask.u64); + lmc_wr(priv, CVMX_LMCX_RODT_MASK(if_num), rodt_mask.u64); + + dyn_rtt_nom_mask = 0; + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + dyn_rtt_nom_mask |= ((rodt_mask.u64 >> (8 * rankx)) & 0xff); + } + if (num_ranks == 4) { + /* + * Normally ODT1 is wired to rank 1. For quad-ranked DIMMs + * ODT1 is wired to the third rank (rank 2). The mask, + * dyn_rtt_nom_mask, is used to indicate for which ranks + * to sweep RTT_NOM during read-leveling. Shift the bit + * from the ODT1 position over to the "ODT2" position so + * that the read-leveling analysis comes out right. + */ + int odt1_bit = dyn_rtt_nom_mask & 2; + + dyn_rtt_nom_mask &= ~2; + dyn_rtt_nom_mask |= odt1_bit << 1; + } + debug("%-45s : 0x%02x\n", "DYN_RTT_NOM_MASK", dyn_rtt_nom_mask); +} + +static void lmc_comp_ctl2(struct ddr_priv *priv) +{ + union cvmx_lmcx_comp_ctl2 cc2; + char *s; + + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + + cc2.cn78xx.dqx_ctl = odt_config[odt_idx].odt_ena; + /* Default 4=34.3 ohm */ + cc2.cn78xx.ck_ctl = (c_cfg->ck_ctl == 0) ? 4 : c_cfg->ck_ctl; + /* Default 4=34.3 ohm */ + cc2.cn78xx.cmd_ctl = (c_cfg->cmd_ctl == 0) ? 4 : c_cfg->cmd_ctl; + /* Default 4=34.3 ohm */ + cc2.cn78xx.control_ctl = (c_cfg->ctl_ctl == 0) ? 4 : c_cfg->ctl_ctl; + + ddr_rodt_ctl_auto = c_cfg->ddr_rodt_ctl_auto; + s = lookup_env(priv, "ddr_rodt_ctl_auto"); + if (s) + ddr_rodt_ctl_auto = !!simple_strtoul(s, NULL, 0); + + default_rodt_ctl = odt_config[odt_idx].qs_dic; + s = lookup_env(priv, "ddr_rodt_ctl"); + if (!s) + s = lookup_env(priv, "ddr%d_rodt_ctl", if_num); + if (s) { + default_rodt_ctl = simple_strtoul(s, NULL, 0); + ddr_rodt_ctl_auto = 0; + } + + cc2.cn70xx.rodt_ctl = default_rodt_ctl; + + // if DDR4, force CK_CTL to 26 ohms if it is currently 34 ohms, + // and DCLK speed is 1 GHz or more... + if (ddr_type == DDR4_DRAM && cc2.s.ck_ctl == ddr4_driver_34_ohm && + ddr_hertz >= 1000000000) { + // lowest for DDR4 is 26 ohms + cc2.s.ck_ctl = ddr4_driver_26_ohm; + debug("N%d.LMC%d: Forcing DDR4 COMP_CTL2[CK_CTL] to %d, %d ohms\n", + node, if_num, cc2.s.ck_ctl, + imp_val->drive_strength[cc2.s.ck_ctl]); + } + + // if DDR4, 2DPC, UDIMM, force CONTROL_CTL and CMD_CTL to 26 ohms, + // if DCLK speed is 1 GHz or more... + if (ddr_type == DDR4_DRAM && dimm_count == 2 && + (spd_dimm_type == 2 || spd_dimm_type == 6) && + ddr_hertz >= 1000000000) { + // lowest for DDR4 is 26 ohms + cc2.cn78xx.control_ctl = ddr4_driver_26_ohm; + // lowest for DDR4 is 26 ohms + cc2.cn78xx.cmd_ctl = ddr4_driver_26_ohm; + debug("N%d.LMC%d: Forcing DDR4 COMP_CTL2[CONTROL_CTL,CMD_CTL] to %d, %d ohms\n", + node, if_num, ddr4_driver_26_ohm, + imp_val->drive_strength[ddr4_driver_26_ohm]); + } + + s = lookup_env(priv, "ddr_ck_ctl"); + if (s) + cc2.cn78xx.ck_ctl = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_cmd_ctl"); + if (s) + cc2.cn78xx.cmd_ctl = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_control_ctl"); + if (s) + cc2.cn70xx.control_ctl = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_dqx_ctl"); + if (s) + cc2.cn78xx.dqx_ctl = simple_strtoul(s, NULL, 0); + + debug("%-45s : %d, %d ohms\n", "DQX_CTL ", cc2.cn78xx.dqx_ctl, + imp_val->drive_strength[cc2.cn78xx.dqx_ctl]); + debug("%-45s : %d, %d ohms\n", "CK_CTL ", cc2.cn78xx.ck_ctl, + imp_val->drive_strength[cc2.cn78xx.ck_ctl]); + debug("%-45s : %d, %d ohms\n", "CMD_CTL ", cc2.cn78xx.cmd_ctl, + imp_val->drive_strength[cc2.cn78xx.cmd_ctl]); + debug("%-45s : %d, %d ohms\n", "CONTROL_CTL ", + cc2.cn78xx.control_ctl, + imp_val->drive_strength[cc2.cn78xx.control_ctl]); + debug("Read ODT_CTL : 0x%x (%d ohms)\n", + cc2.cn78xx.rodt_ctl, imp_val->rodt_ohms[cc2.cn78xx.rodt_ctl]); + + debug("%-45s : 0x%016llx\n", "COMP_CTL2", cc2.u64); + lmc_wr(priv, CVMX_LMCX_COMP_CTL2(if_num), cc2.u64); +} + +static void lmc_phy_ctl(struct ddr_priv *priv) +{ + union cvmx_lmcx_phy_ctl phy_ctl; + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.ts_stagger = 0; + // FIXME: are there others TBD? + phy_ctl.s.dsk_dbg_overwrt_ena = 0; + + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && lranks_per_prank > 1) { + // C0 is TEN, C1 is A17 + phy_ctl.s.c0_sel = 2; + phy_ctl.s.c1_sel = 2; + debug("N%d.LMC%d: 3DS: setting PHY_CTL[cx_csel] = %d\n", + node, if_num, phy_ctl.s.c1_sel); + } + + debug("PHY_CTL : 0x%016llx\n", + phy_ctl.u64); + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); +} + +static void lmc_ext_config(struct ddr_priv *priv) +{ + union cvmx_lmcx_ext_config ext_cfg; + char *s; + + ext_cfg.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num)); + ext_cfg.s.vrefint_seq_deskew = 0; + ext_cfg.s.read_ena_bprch = 1; + ext_cfg.s.read_ena_fprch = 1; + ext_cfg.s.drive_ena_fprch = 1; + ext_cfg.s.drive_ena_bprch = 1; + // make sure this is OFF for all current chips + ext_cfg.s.invert_data = 0; + + s = lookup_env(priv, "ddr_read_fprch"); + if (s) + ext_cfg.s.read_ena_fprch = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_read_bprch"); + if (s) + ext_cfg.s.read_ena_bprch = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_drive_fprch"); + if (s) + ext_cfg.s.drive_ena_fprch = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_drive_bprch"); + if (s) + ext_cfg.s.drive_ena_bprch = strtoul(s, NULL, 0); + + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && lranks_per_prank > 1) { + ext_cfg.s.dimm0_cid = lranks_bits; + ext_cfg.s.dimm1_cid = lranks_bits; + debug("N%d.LMC%d: 3DS: setting EXT_CONFIG[dimmx_cid] = %d\n", + node, if_num, ext_cfg.s.dimm0_cid); + } + + lmc_wr(priv, CVMX_LMCX_EXT_CONFIG(if_num), ext_cfg.u64); + debug("%-45s : 0x%016llx\n", "EXT_CONFIG", ext_cfg.u64); +} + +static void lmc_ext_config2(struct ddr_priv *priv) +{ + char *s; + + // NOTE: all chips have this register, but not necessarily the + // fields we modify... + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && + !octeon_is_cpuid(OCTEON_CN73XX)) { + union cvmx_lmcx_ext_config2 ext_cfg2; + int value = 1; // default to 1 + + ext_cfg2.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG2(if_num)); + + s = lookup_env(priv, "ddr_ext2_delay_unload"); + if (s) + value = !!simple_strtoul(s, NULL, 0); + + ext_cfg2.s.delay_unload_r0 = value; + ext_cfg2.s.delay_unload_r1 = value; + ext_cfg2.s.delay_unload_r2 = value; + ext_cfg2.s.delay_unload_r3 = value; + + lmc_wr(priv, CVMX_LMCX_EXT_CONFIG2(if_num), ext_cfg2.u64); + debug("%-45s : 0x%016llx\n", "EXT_CONFIG2", ext_cfg2.u64); + } +} + +static void lmc_dimm01_params_loop(struct ddr_priv *priv) +{ + union cvmx_lmcx_dimmx_params dimm_p; + int dimmx = didx; + char *s; + int rc; + int i; + + dimm_p.u64 = lmc_rd(priv, CVMX_LMCX_DIMMX_PARAMS(dimmx, if_num)); + + if (ddr_type == DDR4_DRAM) { + union cvmx_lmcx_dimmx_ddr4_params0 ddr4_p0; + union cvmx_lmcx_dimmx_ddr4_params1 ddr4_p1; + union cvmx_lmcx_ddr4_dimm_ctl ddr4_ctl; + + dimm_p.s.rc0 = 0; + dimm_p.s.rc1 = 0; + dimm_p.s.rc2 = 0; + + rc = read_spd(&dimm_config_table[didx], 0, + DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CTL); + dimm_p.s.rc3 = (rc >> 4) & 0xf; + dimm_p.s.rc4 = ((rc >> 0) & 0x3) << 2; + dimm_p.s.rc4 |= ((rc >> 2) & 0x3) << 0; + + rc = read_spd(&dimm_config_table[didx], 0, + DDR4_SPD_RDIMM_REGISTER_DRIVE_STRENGTH_CK); + dimm_p.s.rc5 = ((rc >> 0) & 0x3) << 2; + dimm_p.s.rc5 |= ((rc >> 2) & 0x3) << 0; + + dimm_p.s.rc6 = 0; + dimm_p.s.rc7 = 0; + dimm_p.s.rc8 = 0; + dimm_p.s.rc9 = 0; + + /* + * rc10 DDR4 RDIMM Operating Speed + * === =================================================== + * 0 tclk_psecs >= 1250 psec DDR4-1600 (1250 ps) + * 1 1250 psec > tclk_psecs >= 1071 psec DDR4-1866 (1071 ps) + * 2 1071 psec > tclk_psecs >= 938 psec DDR4-2133 ( 938 ps) + * 3 938 psec > tclk_psecs >= 833 psec DDR4-2400 ( 833 ps) + * 4 833 psec > tclk_psecs >= 750 psec DDR4-2666 ( 750 ps) + * 5 750 psec > tclk_psecs >= 625 psec DDR4-3200 ( 625 ps) + */ + dimm_p.s.rc10 = 0; + if (tclk_psecs < 1250) + dimm_p.s.rc10 = 1; + if (tclk_psecs < 1071) + dimm_p.s.rc10 = 2; + if (tclk_psecs < 938) + dimm_p.s.rc10 = 3; + if (tclk_psecs < 833) + dimm_p.s.rc10 = 4; + if (tclk_psecs < 750) + dimm_p.s.rc10 = 5; + + dimm_p.s.rc11 = 0; + dimm_p.s.rc12 = 0; + /* 0=LRDIMM, 1=RDIMM */ + dimm_p.s.rc13 = (spd_dimm_type == 4) ? 0 : 4; + dimm_p.s.rc13 |= (ddr_type == DDR4_DRAM) ? + (spd_addr_mirror << 3) : 0; + dimm_p.s.rc14 = 0; + dimm_p.s.rc15 = 0; /* 1 nCK latency adder */ + + ddr4_p0.u64 = 0; + + ddr4_p0.s.rc8x = 0; + ddr4_p0.s.rc7x = 0; + ddr4_p0.s.rc6x = 0; + ddr4_p0.s.rc5x = 0; + ddr4_p0.s.rc4x = 0; + + ddr4_p0.s.rc3x = compute_rc3x(tclk_psecs); + + ddr4_p0.s.rc2x = 0; + ddr4_p0.s.rc1x = 0; + + ddr4_p1.u64 = 0; + + ddr4_p1.s.rcbx = 0; + ddr4_p1.s.rcax = 0; + ddr4_p1.s.rc9x = 0; + + ddr4_ctl.u64 = 0; + ddr4_ctl.cn70xx.ddr4_dimm0_wmask = 0x004; + ddr4_ctl.cn70xx.ddr4_dimm1_wmask = + (dimm_count > 1) ? 0x004 : 0x0000; + + /* + * Handle any overrides from envvars here... + */ + s = lookup_env(priv, "ddr_ddr4_params0"); + if (s) + ddr4_p0.u64 = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_ddr4_params1"); + if (s) + ddr4_p1.u64 = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_ddr4_dimm_ctl"); + if (s) + ddr4_ctl.u64 = simple_strtoul(s, NULL, 0); + + for (i = 0; i < 11; ++i) { + u64 value; + + s = lookup_env(priv, "ddr_ddr4_rc%1xx", i + 1); + if (s) { + value = simple_strtoul(s, NULL, 0); + if (i < 8) { + ddr4_p0.u64 &= ~((u64)0xff << (i * 8)); + ddr4_p0.u64 |= (value << (i * 8)); + } else { + ddr4_p1.u64 &= + ~((u64)0xff << ((i - 8) * 8)); + ddr4_p1.u64 |= (value << ((i - 8) * 8)); + } + } + } + + /* + * write the final CSR values + */ + lmc_wr(priv, CVMX_LMCX_DIMMX_DDR4_PARAMS0(dimmx, if_num), + ddr4_p0.u64); + + lmc_wr(priv, CVMX_LMCX_DDR4_DIMM_CTL(if_num), ddr4_ctl.u64); + + lmc_wr(priv, CVMX_LMCX_DIMMX_DDR4_PARAMS1(dimmx, if_num), + ddr4_p1.u64); + + debug("DIMM%d Register Control Words RCBx:RC1x : %x %x %x %x %x %x %x %x %x %x %x\n", + dimmx, ddr4_p1.s.rcbx, ddr4_p1.s.rcax, + ddr4_p1.s.rc9x, ddr4_p0.s.rc8x, + ddr4_p0.s.rc7x, ddr4_p0.s.rc6x, + ddr4_p0.s.rc5x, ddr4_p0.s.rc4x, + ddr4_p0.s.rc3x, ddr4_p0.s.rc2x, ddr4_p0.s.rc1x); + + } else { + rc = read_spd(&dimm_config_table[didx], 0, 69); + dimm_p.s.rc0 = (rc >> 0) & 0xf; + dimm_p.s.rc1 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 70); + dimm_p.s.rc2 = (rc >> 0) & 0xf; + dimm_p.s.rc3 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 71); + dimm_p.s.rc4 = (rc >> 0) & 0xf; + dimm_p.s.rc5 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 72); + dimm_p.s.rc6 = (rc >> 0) & 0xf; + dimm_p.s.rc7 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 73); + dimm_p.s.rc8 = (rc >> 0) & 0xf; + dimm_p.s.rc9 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 74); + dimm_p.s.rc10 = (rc >> 0) & 0xf; + dimm_p.s.rc11 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 75); + dimm_p.s.rc12 = (rc >> 0) & 0xf; + dimm_p.s.rc13 = (rc >> 4) & 0xf; + + rc = read_spd(&dimm_config_table[didx], 0, 76); + dimm_p.s.rc14 = (rc >> 0) & 0xf; + dimm_p.s.rc15 = (rc >> 4) & 0xf; + + s = ddr_getenv_debug(priv, "ddr_clk_drive"); + if (s) { + if (strcmp(s, "light") == 0) + dimm_p.s.rc5 = 0x0; /* Light Drive */ + if (strcmp(s, "moderate") == 0) + dimm_p.s.rc5 = 0x5; /* Moderate Drive */ + if (strcmp(s, "strong") == 0) + dimm_p.s.rc5 = 0xA; /* Strong Drive */ + printf("Parameter found in environment. ddr_clk_drive = %s\n", + s); + } + + s = ddr_getenv_debug(priv, "ddr_cmd_drive"); + if (s) { + if (strcmp(s, "light") == 0) + dimm_p.s.rc3 = 0x0; /* Light Drive */ + if (strcmp(s, "moderate") == 0) + dimm_p.s.rc3 = 0x5; /* Moderate Drive */ + if (strcmp(s, "strong") == 0) + dimm_p.s.rc3 = 0xA; /* Strong Drive */ + printf("Parameter found in environment. ddr_cmd_drive = %s\n", + s); + } + + s = ddr_getenv_debug(priv, "ddr_ctl_drive"); + if (s) { + if (strcmp(s, "light") == 0) + dimm_p.s.rc4 = 0x0; /* Light Drive */ + if (strcmp(s, "moderate") == 0) + dimm_p.s.rc4 = 0x5; /* Moderate Drive */ + printf("Parameter found in environment. ddr_ctl_drive = %s\n", + s); + } + + /* + * rc10 DDR3 RDIMM Operating Speed + * == ===================================================== + * 0 tclk_psecs >= 2500 psec DDR3/DDR3L-800 def + * 1 2500 psec > tclk_psecs >= 1875 psec DDR3/DDR3L-1066 + * 2 1875 psec > tclk_psecs >= 1500 psec DDR3/DDR3L-1333 + * 3 1500 psec > tclk_psecs >= 1250 psec DDR3/DDR3L-1600 + * 4 1250 psec > tclk_psecs >= 1071 psec DDR3-1866 + */ + dimm_p.s.rc10 = 0; + if (tclk_psecs < 2500) + dimm_p.s.rc10 = 1; + if (tclk_psecs < 1875) + dimm_p.s.rc10 = 2; + if (tclk_psecs < 1500) + dimm_p.s.rc10 = 3; + if (tclk_psecs < 1250) + dimm_p.s.rc10 = 4; + } + + s = lookup_env(priv, "ddr_dimmx_params", i); + if (s) + dimm_p.u64 = simple_strtoul(s, NULL, 0); + + for (i = 0; i < 16; ++i) { + u64 value; + + s = lookup_env(priv, "ddr_rc%d", i); + if (s) { + value = simple_strtoul(s, NULL, 0); + dimm_p.u64 &= ~((u64)0xf << (i * 4)); + dimm_p.u64 |= (value << (i * 4)); + } + } + + lmc_wr(priv, CVMX_LMCX_DIMMX_PARAMS(dimmx, if_num), dimm_p.u64); + + debug("DIMM%d Register Control Words RC15:RC0 : %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x\n", + dimmx, dimm_p.s.rc15, dimm_p.s.rc14, dimm_p.s.rc13, + dimm_p.s.rc12, dimm_p.s.rc11, dimm_p.s.rc10, + dimm_p.s.rc9, dimm_p.s.rc8, dimm_p.s.rc7, + dimm_p.s.rc6, dimm_p.s.rc5, dimm_p.s.rc4, + dimm_p.s.rc3, dimm_p.s.rc2, dimm_p.s.rc1, dimm_p.s.rc0); + + // FIXME: recognize a DDR3 RDIMM with 4 ranks and 2 registers, + // and treat it specially + if (ddr_type == DDR3_DRAM && num_ranks == 4 && + spd_rdimm_registers == 2 && dimmx == 0) { + debug("DDR3: Copying DIMM0_PARAMS to DIMM1_PARAMS for pseudo-DIMM #1...\n"); + lmc_wr(priv, CVMX_LMCX_DIMMX_PARAMS(1, if_num), dimm_p.u64); + } +} + +static void lmc_dimm01_params(struct ddr_priv *priv) +{ + union cvmx_lmcx_dimm_ctl dimm_ctl; + char *s; + + if (spd_rdimm) { + for (didx = 0; didx < (unsigned int)dimm_count; ++didx) + lmc_dimm01_params_loop(priv); + + if (ddr_type == DDR4_DRAM) { + /* LMC0_DIMM_CTL */ + dimm_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DIMM_CTL(if_num)); + dimm_ctl.s.dimm0_wmask = 0xdf3f; + dimm_ctl.s.dimm1_wmask = + (dimm_count > 1) ? 0xdf3f : 0x0000; + dimm_ctl.s.tcws = 0x4e0; + dimm_ctl.s.parity = c_cfg->parity; + + s = lookup_env(priv, "ddr_dimm0_wmask"); + if (s) { + dimm_ctl.s.dimm0_wmask = + simple_strtoul(s, NULL, 0); + } + + s = lookup_env(priv, "ddr_dimm1_wmask"); + if (s) { + dimm_ctl.s.dimm1_wmask = + simple_strtoul(s, NULL, 0); + } + + s = lookup_env(priv, "ddr_dimm_ctl_parity"); + if (s) + dimm_ctl.s.parity = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_dimm_ctl_tcws"); + if (s) + dimm_ctl.s.tcws = simple_strtoul(s, NULL, 0); + + debug("LMC DIMM_CTL : 0x%016llx\n", + dimm_ctl.u64); + lmc_wr(priv, CVMX_LMCX_DIMM_CTL(if_num), dimm_ctl.u64); + + /* Init RCW */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x7); + + /* Write RC0D last */ + dimm_ctl.s.dimm0_wmask = 0x2000; + dimm_ctl.s.dimm1_wmask = (dimm_count > 1) ? + 0x2000 : 0x0000; + debug("LMC DIMM_CTL : 0x%016llx\n", + dimm_ctl.u64); + lmc_wr(priv, CVMX_LMCX_DIMM_CTL(if_num), dimm_ctl.u64); + + /* + * Don't write any extended registers the second time + */ + lmc_wr(priv, CVMX_LMCX_DDR4_DIMM_CTL(if_num), 0); + + /* Init RCW */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x7); + } else { + /* LMC0_DIMM_CTL */ + dimm_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DIMM_CTL(if_num)); + dimm_ctl.s.dimm0_wmask = 0xffff; + // FIXME: recognize a DDR3 RDIMM with 4 ranks and 2 + // registers, and treat it specially + if (num_ranks == 4 && spd_rdimm_registers == 2) { + debug("DDR3: Activating DIMM_CTL[dimm1_mask] bits...\n"); + dimm_ctl.s.dimm1_wmask = 0xffff; + } else { + dimm_ctl.s.dimm1_wmask = + (dimm_count > 1) ? 0xffff : 0x0000; + } + dimm_ctl.s.tcws = 0x4e0; + dimm_ctl.s.parity = c_cfg->parity; + + s = lookup_env(priv, "ddr_dimm0_wmask"); + if (s) { + dimm_ctl.s.dimm0_wmask = + simple_strtoul(s, NULL, 0); + } + + s = lookup_env(priv, "ddr_dimm1_wmask"); + if (s) { + dimm_ctl.s.dimm1_wmask = + simple_strtoul(s, NULL, 0); + } + + s = lookup_env(priv, "ddr_dimm_ctl_parity"); + if (s) + dimm_ctl.s.parity = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_dimm_ctl_tcws"); + if (s) + dimm_ctl.s.tcws = simple_strtoul(s, NULL, 0); + + debug("LMC DIMM_CTL : 0x%016llx\n", + dimm_ctl.u64); + lmc_wr(priv, CVMX_LMCX_DIMM_CTL(if_num), dimm_ctl.u64); + + /* Init RCW */ + oct3_ddr3_seq(priv, rank_mask, if_num, 0x7); + } + + } else { + /* Disable register control writes for unbuffered */ + union cvmx_lmcx_dimm_ctl dimm_ctl; + + dimm_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DIMM_CTL(if_num)); + dimm_ctl.s.dimm0_wmask = 0; + dimm_ctl.s.dimm1_wmask = 0; + lmc_wr(priv, CVMX_LMCX_DIMM_CTL(if_num), dimm_ctl.u64); + } +} + +static int lmc_rank_init(struct ddr_priv *priv) +{ + char *s; + +#if ALLOW_BY_RANK_INIT + if (enable_by_rank_init) { + by_rank = 3; + saved_rank_mask = rank_mask; + } + +start_by_rank_init: + + if (enable_by_rank_init) { + rank_mask = (1 << by_rank); + if (!(rank_mask & saved_rank_mask)) + goto end_by_rank_init; + if (by_rank == 0) + rank_mask = saved_rank_mask; + + debug("\n>>>>> BY_RANK: starting rank %d with mask 0x%02x\n\n", + by_rank, rank_mask); + } + +#endif /* ALLOW_BY_RANK_INIT */ + + /* + * Comments (steps 3 through 5) continue in oct3_ddr3_seq() + */ + union cvmx_lmcx_modereg_params0 mp0; + + if (ddr_memory_preserved(priv)) { + /* + * Contents are being preserved. Take DRAM out of self-refresh + * first. Then init steps can procede normally + */ + /* self-refresh exit */ + oct3_ddr3_seq(priv, rank_mask, if_num, 3); + } + + mp0.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num)); + mp0.s.dllr = 1; /* Set during first init sequence */ + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num), mp0.u64); + + ddr_init_seq(priv, rank_mask, if_num); + + mp0.s.dllr = 0; /* Clear for normal operation */ + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num), mp0.u64); + + if (spd_rdimm && ddr_type == DDR4_DRAM && + octeon_is_cpuid(OCTEON_CN7XXX)) { + debug("Running init sequence 1\n"); + change_rdimm_mpr_pattern(priv, rank_mask, if_num, dimm_count); + } + + memset(lanes, 0, sizeof(lanes)); + for (lane = 0; lane < last_lane; lane++) { + // init all lanes to reset value + dac_settings[lane] = 127; + } + + // FIXME: disable internal VREF if deskew is disabled? + if (disable_deskew_training) { + debug("N%d.LMC%d: internal VREF Training disabled, leaving them in RESET.\n", + node, if_num); + num_samples = 0; + } else if (ddr_type == DDR4_DRAM && + !octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) { + num_samples = DEFAULT_DAC_SAMPLES; + } else { + // if DDR3 or no ability to write DAC values + num_samples = 1; + } + +perform_internal_vref_training: + + total_dac_eval_retries = 0; + dac_eval_exhausted = 0; + + for (sample = 0; sample < num_samples; sample++) { + dac_eval_retries = 0; + + // make offset and internal vref training repeatable + do { + /* + * 6.9.8 LMC Offset Training + * LMC requires input-receiver offset training. + */ + perform_offset_training(priv, rank_mask, if_num); + + /* + * 6.9.9 LMC Internal vref Training + * LMC requires input-reference-voltage training. + */ + perform_internal_vref_training(priv, rank_mask, if_num); + + // read and maybe display the DAC values for a sample + read_dac_dbi_settings(priv, if_num, /*DAC*/ 1, + dac_settings); + if (num_samples == 1 || ddr_verbose(priv)) { + display_dac_dbi_settings(if_num, /*DAC*/ 1, + use_ecc, dac_settings, + "Internal VREF"); + } + + // for DDR4, evaluate the DAC settings and retry + // if any issues + if (ddr_type == DDR4_DRAM) { + if (evaluate_dac_settings + (if_64b, use_ecc, dac_settings)) { + dac_eval_retries += 1; + if (dac_eval_retries > + DAC_RETRIES_LIMIT) { + debug("N%d.LMC%d: DDR4 internal VREF DAC settings: retries exhausted; continuing...\n", + node, if_num); + dac_eval_exhausted += 1; + } else { + debug("N%d.LMC%d: DDR4 internal VREF DAC settings inconsistent; retrying....\n", + node, if_num); + total_dac_eval_retries += 1; + // try another sample + continue; + } + } + + // taking multiple samples, otherwise do nothing + if (num_samples > 1) { + // good sample or exhausted retries, + // record it + for (lane = 0; lane < last_lane; + lane++) { + lanes[lane].bytes[sample] = + dac_settings[lane]; + } + } + } + // done if DDR3, or good sample, or exhausted retries + break; + } while (1); + } + + if (ddr_type == DDR4_DRAM && dac_eval_exhausted > 0) { + debug("N%d.LMC%d: DDR internal VREF DAC settings: total retries %d, exhausted %d\n", + node, if_num, total_dac_eval_retries, dac_eval_exhausted); + } + + if (num_samples > 1) { + debug("N%d.LMC%d: DDR4 internal VREF DAC settings: processing multiple samples...\n", + node, if_num); + + for (lane = 0; lane < last_lane; lane++) { + dac_settings[lane] = + process_samples_average(&lanes[lane].bytes[0], + num_samples, if_num, lane); + } + display_dac_dbi_settings(if_num, /*DAC*/ 1, use_ecc, + dac_settings, "Averaged VREF"); + + // finally, write the final DAC values + for (lane = 0; lane < last_lane; lane++) { + load_dac_override(priv, if_num, dac_settings[lane], + lane); + } + } + + // allow override of any byte-lane internal VREF + int overrode_vref_dac = 0; + + for (lane = 0; lane < last_lane; lane++) { + s = lookup_env(priv, "ddr%d_vref_dac_byte%d", if_num, lane); + if (s) { + dac_settings[lane] = simple_strtoul(s, NULL, 0); + overrode_vref_dac = 1; + // finally, write the new DAC value + load_dac_override(priv, if_num, dac_settings[lane], + lane); + } + } + if (overrode_vref_dac) { + display_dac_dbi_settings(if_num, /*DAC*/ 1, use_ecc, + dac_settings, "Override VREF"); + } + +#if DDR3_DAC_OVERRIDE + // as a second step, after internal VREF training, before starting + // deskew training: + // for DDR3 and OCTEON3 not O78 pass 1.x, override the DAC setting + // to 127 + if (ddr_type == DDR3_DRAM && !octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) && + !disable_deskew_training) { + load_dac_override(priv, if_num, 127, /* all */ 0x0A); + debug("N%d.LMC%d: Overriding DDR3 internal VREF DAC settings to 127.\n", + node, if_num); + } +#endif + + /* + * 4.8.8 LMC Deskew Training + * + * LMC requires input-read-data deskew training. + */ + if (!disable_deskew_training) { + deskew_training_errors = + perform_deskew_training(priv, rank_mask, if_num, + spd_rawcard_aorb); + + // All the Deskew lock and saturation retries (may) have + // been done, but we ended up with nibble errors; so, + // as a last ditch effort, try the Internal vref + // Training again... + if (deskew_training_errors) { + if (internal_retries < + DEFAULT_INTERNAL_VREF_TRAINING_LIMIT) { + internal_retries++; + debug("N%d.LMC%d: Deskew training results still unsettled - retrying internal vref training (%d)\n", + node, if_num, internal_retries); + goto perform_internal_vref_training; + } else { + if (restart_if_dsk_incomplete) { + debug("N%d.LMC%d: INFO: Deskew training incomplete - %d retries exhausted, Restarting LMC init...\n", + node, if_num, internal_retries); + return -EAGAIN; + } + debug("N%d.LMC%d: Deskew training incomplete - %d retries exhausted, but continuing...\n", + node, if_num, internal_retries); + } + } /* if (deskew_training_errors) */ + + // FIXME: treat this as the final DSK print from now on, + // and print if VBL_NORM or above also, save the results + // of the original training in case we want them later + validate_deskew_training(priv, rank_mask, if_num, + &deskew_training_results, 1); + } else { /* if (! disable_deskew_training) */ + debug("N%d.LMC%d: Deskew Training disabled, printing settings before HWL.\n", + node, if_num); + validate_deskew_training(priv, rank_mask, if_num, + &deskew_training_results, 1); + } /* if (! disable_deskew_training) */ + +#if ALLOW_BY_RANK_INIT + + if (enable_by_rank_init) { + read_dac_dbi_settings(priv, if_num, /*dac */ 1, + &rank_dac[by_rank].bytes[0]); + get_deskew_settings(priv, if_num, &rank_dsk[by_rank]); + debug("\n>>>>> BY_RANK: ending rank %d\n\n", by_rank); + } + +end_by_rank_init: + + if (enable_by_rank_init) { + //debug("\n>>>>> BY_RANK: ending rank %d\n\n", by_rank); + + by_rank--; + if (by_rank >= 0) + goto start_by_rank_init; + + rank_mask = saved_rank_mask; + ddr_init_seq(priv, rank_mask, if_num); + + process_by_rank_dac(priv, if_num, rank_mask, rank_dac); + process_by_rank_dsk(priv, if_num, rank_mask, rank_dsk); + + // FIXME: set this to prevent later checking!!! + disable_deskew_training = 1; + + debug("\n>>>>> BY_RANK: FINISHED!!\n\n"); + } +#endif /* ALLOW_BY_RANK_INIT */ + + return 0; +} + +static void lmc_config_2(struct ddr_priv *priv) +{ + union cvmx_lmcx_config lmc_config; + int save_ref_zqcs_int; + u64 temp_delay_usecs; + + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + + /* + * Temporarily select the minimum ZQCS interval and wait + * long enough for a few ZQCS calibrations to occur. This + * should ensure that the calibration circuitry is + * stabilized before read/write leveling occurs. + */ + if (octeon_is_cpuid(OCTEON_CN7XXX)) { + save_ref_zqcs_int = lmc_config.cn78xx.ref_zqcs_int; + /* set smallest interval */ + lmc_config.cn78xx.ref_zqcs_int = 1 | (32 << 7); + } else { + save_ref_zqcs_int = lmc_config.cn63xx.ref_zqcs_int; + /* set smallest interval */ + lmc_config.cn63xx.ref_zqcs_int = 1 | (32 << 7); + } + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64); + lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + + /* + * Compute an appropriate delay based on the current ZQCS + * interval. The delay should be long enough for the + * current ZQCS delay counter to expire plus ten of the + * minimum intarvals to ensure that some calibrations + * occur. + */ + temp_delay_usecs = (((u64)save_ref_zqcs_int >> 7) * tclk_psecs * + 100 * 512 * 128) / (10000 * 10000) + 10 * + ((u64)32 * tclk_psecs * 100 * 512 * 128) / (10000 * 10000); + + debug("Waiting %lld usecs for ZQCS calibrations to start\n", + temp_delay_usecs); + udelay(temp_delay_usecs); + + if (octeon_is_cpuid(OCTEON_CN7XXX)) { + /* Restore computed interval */ + lmc_config.cn78xx.ref_zqcs_int = save_ref_zqcs_int; + } else { + /* Restore computed interval */ + lmc_config.cn63xx.ref_zqcs_int = save_ref_zqcs_int; + } + + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), lmc_config.u64); + lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); +} + +static union cvmx_lmcx_wlevel_ctl wl_ctl __section(".data"); +static union cvmx_lmcx_wlevel_rankx wl_rank __section(".data"); +static union cvmx_lmcx_modereg_params1 mp1 __section(".data"); + +static int wl_mask[9] __section(".data"); +static int byte_idx __section(".data"); +static int ecc_ena __section(".data"); +static int wl_roundup __section(".data"); +static int save_mode32b __section(".data"); +static int disable_hwl_validity __section(".data"); +static int default_wl_rtt_nom __section(".data"); +static int wl_pbm_pump __section(".data"); + +static void lmc_write_leveling_loop(struct ddr_priv *priv, int rankx) +{ + int wloop = 0; + // retries per sample for HW-related issues with bitmasks or values + int wloop_retries = 0; + int wloop_retries_total = 0; + int wloop_retries_exhausted = 0; +#define WLOOP_RETRIES_DEFAULT 5 + int wl_val_err; + int wl_mask_err_rank = 0; + int wl_val_err_rank = 0; +#if HW_WL_MAJORITY + // array to collect counts of byte-lane values + // assume low-order 3 bits and even, so really only 2-bit values + struct wlevel_bitcnt wl_bytes[9], wl_bytes_extra[9]; + int extra_bumps, extra_mask; +#endif + int rank_nom = 0; + + if (!(rank_mask & (1 << rankx))) + return; + + if (match_wl_rtt_nom) { + if (rankx == 0) + rank_nom = mp1.s.rtt_nom_00; + if (rankx == 1) + rank_nom = mp1.s.rtt_nom_01; + if (rankx == 2) + rank_nom = mp1.s.rtt_nom_10; + if (rankx == 3) + rank_nom = mp1.s.rtt_nom_11; + + debug("N%d.LMC%d.R%d: Setting WLEVEL_CTL[rtt_nom] to %d (%d)\n", + node, if_num, rankx, rank_nom, + imp_val->rtt_nom_ohms[rank_nom]); + } + +#if HW_WL_MAJORITY + memset(wl_bytes, 0, sizeof(wl_bytes)); + memset(wl_bytes_extra, 0, sizeof(wl_bytes_extra)); +#endif + + // restructure the looping so we can keep trying until we get the + // samples we want + while (wloop < wl_loops) { + wl_ctl.u64 = lmc_rd(priv, CVMX_LMCX_WLEVEL_CTL(if_num)); + + wl_ctl.cn78xx.rtt_nom = + (default_wl_rtt_nom > 0) ? (default_wl_rtt_nom - 1) : 7; + + if (match_wl_rtt_nom) { + wl_ctl.cn78xx.rtt_nom = + (rank_nom > 0) ? (rank_nom - 1) : 7; + } + + /* Clear write-level delays */ + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), 0); + + wl_mask_err = 0; /* Reset error counters */ + wl_val_err = 0; + + for (byte_idx = 0; byte_idx < 9; ++byte_idx) + wl_mask[byte_idx] = 0; /* Reset bitmasks */ + + // do all the byte-lanes at the same time + wl_ctl.cn78xx.lanemask = 0x1ff; + + lmc_wr(priv, CVMX_LMCX_WLEVEL_CTL(if_num), wl_ctl.u64); + + /* + * Read and write values back in order to update the + * status field. This insures that we read the updated + * values after write-leveling has completed. + */ + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), + lmc_rd(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num))); + + /* write-leveling */ + oct3_ddr3_seq(priv, 1 << rankx, if_num, 6); + + do { + wl_rank.u64 = lmc_rd(priv, + CVMX_LMCX_WLEVEL_RANKX(rankx, + if_num)); + } while (wl_rank.cn78xx.status != 3); + + wl_rank.u64 = lmc_rd(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, + if_num)); + + for (byte_idx = 0; byte_idx < (8 + ecc_ena); ++byte_idx) { + wl_mask[byte_idx] = lmc_ddr3_wl_dbg_read(priv, + if_num, + byte_idx); + if (wl_mask[byte_idx] == 0) + ++wl_mask_err; + } + + // check validity only if no bitmask errors + if (wl_mask_err == 0) { + if ((spd_dimm_type == 1 || spd_dimm_type == 2) && + dram_width != 16 && if_64b && + !disable_hwl_validity) { + // bypass if [mini|SO]-[RU]DIMM or x16 or + // 32-bit + wl_val_err = + validate_hw_wl_settings(if_num, + &wl_rank, + spd_rdimm, ecc_ena); + wl_val_err_rank += (wl_val_err != 0); + } + } else { + wl_mask_err_rank++; + } + + // before we print, if we had bitmask or validity errors, + // do a retry... + if (wl_mask_err != 0 || wl_val_err != 0) { + if (wloop_retries < WLOOP_RETRIES_DEFAULT) { + wloop_retries++; + wloop_retries_total++; + // this printout is per-retry: only when VBL + // is high enough (DEV?) + // FIXME: do we want to show the bad bitmaps + // or delays here also? + debug("N%d.LMC%d.R%d: H/W Write-Leveling had %s errors - retrying...\n", + node, if_num, rankx, + (wl_mask_err) ? "Bitmask" : "Validity"); + // this takes us back to the top without + // counting a sample + return; + } + + // retries exhausted, do not print at normal VBL + debug("N%d.LMC%d.R%d: H/W Write-Leveling issues: %s errors\n", + node, if_num, rankx, + (wl_mask_err) ? "Bitmask" : "Validity"); + wloop_retries_exhausted++; + } + // no errors or exhausted retries, use this sample + wloop_retries = 0; //reset for next sample + + // when only 1 sample or forced, print the bitmasks then + // current HW WL + if (wl_loops == 1 || wl_print) { + if (wl_print > 1) + display_wl_bm(if_num, rankx, wl_mask); + display_wl(if_num, wl_rank, rankx); + } + + if (wl_roundup) { /* Round up odd bitmask delays */ + for (byte_idx = 0; byte_idx < (8 + ecc_ena); + ++byte_idx) { + if (!(if_bytemask & (1 << byte_idx))) + return; + upd_wl_rank(&wl_rank, byte_idx, + roundup_ddr3_wlevel_bitmask + (wl_mask[byte_idx])); + } + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), + wl_rank.u64); + display_wl(if_num, wl_rank, rankx); + } + +#if HW_WL_MAJORITY + // OK, we have a decent sample, no bitmask or validity errors + extra_bumps = 0; + extra_mask = 0; + for (byte_idx = 0; byte_idx < (8 + ecc_ena); ++byte_idx) { + int ix; + + if (!(if_bytemask & (1 << byte_idx))) + return; + + // increment count of byte-lane value + // only 4 values + ix = (get_wl_rank(&wl_rank, byte_idx) >> 1) & 3; + wl_bytes[byte_idx].bitcnt[ix]++; + wl_bytes_extra[byte_idx].bitcnt[ix]++; + // if perfect... + if (__builtin_popcount(wl_mask[byte_idx]) == 4) { + wl_bytes_extra[byte_idx].bitcnt[ix] += + wl_pbm_pump; + extra_bumps++; + extra_mask |= 1 << byte_idx; + } + } + + if (extra_bumps) { + if (wl_print > 1) { + debug("N%d.LMC%d.R%d: HWL sample had %d bumps (0x%02x).\n", + node, if_num, rankx, extra_bumps, + extra_mask); + } + } +#endif + + // if we get here, we have taken a decent sample + wloop++; + + } /* while (wloop < wl_loops) */ + +#if HW_WL_MAJORITY + // if we did sample more than once, try to pick a majority vote + if (wl_loops > 1) { + // look for the majority in each byte-lane + for (byte_idx = 0; byte_idx < (8 + ecc_ena); ++byte_idx) { + int mx, mc, xc, cc; + int ix, alts; + int maj, xmaj, xmx, xmc, xxc, xcc; + + if (!(if_bytemask & (1 << byte_idx))) + return; + maj = find_wl_majority(&wl_bytes[byte_idx], &mx, + &mc, &xc, &cc); + xmaj = find_wl_majority(&wl_bytes_extra[byte_idx], + &xmx, &xmc, &xxc, &xcc); + if (maj != xmaj) { + if (wl_print) { + debug("N%d.LMC%d.R%d: Byte %d: HWL maj %d(%d), USING xmaj %d(%d)\n", + node, if_num, rankx, + byte_idx, maj, xc, xmaj, xxc); + } + mx = xmx; + mc = xmc; + xc = xxc; + cc = xcc; + } + +#if SWL_TRY_HWL_ALT + // see if there was an alternate + // take out the majority choice + alts = (mc & ~(1 << mx)); + if (alts != 0) { + for (ix = 0; ix < 4; ix++) { + // FIXME: could be done multiple times? + // bad if so + if (alts & (1 << ix)) { + // set the mask + hwl_alts[rankx].hwl_alt_mask |= + (1 << byte_idx); + // record the value + hwl_alts[rankx].hwl_alt_delay[byte_idx] = + ix << 1; + if (wl_print > 1) { + debug("N%d.LMC%d.R%d: SWL_TRY_HWL_ALT: Byte %d maj %d (%d) alt %d (%d).\n", + node, + if_num, + rankx, + byte_idx, + mx << 1, + xc, + ix << 1, + wl_bytes + [byte_idx].bitcnt + [ix]); + } + } + } + } +#endif /* SWL_TRY_HWL_ALT */ + + if (cc > 2) { // unlikely, but... + // assume: counts for 3 indices are all 1 + // possiblities are: 0/2/4, 2/4/6, 0/4/6, 0/2/6 + // and the desired?: 2 , 4 , 6, 0 + // we choose the middle, assuming one of the + // outliers is bad + // NOTE: this is an ugly hack at the moment; + // there must be a better way + switch (mc) { + case 0x7: + mx = 1; + break; // was 0/2/4, choose 2 + case 0xb: + mx = 0; + break; // was 0/2/6, choose 0 + case 0xd: + mx = 3; + break; // was 0/4/6, choose 6 + case 0xe: + mx = 2; + break; // was 2/4/6, choose 4 + default: + case 0xf: + mx = 1; + break; // was 0/2/4/6, choose 2? + } + printf("N%d.LMC%d.R%d: HW WL MAJORITY: bad byte-lane %d (0x%x), using %d.\n", + node, if_num, rankx, byte_idx, mc, + mx << 1); + } + upd_wl_rank(&wl_rank, byte_idx, mx << 1); + } + + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), + wl_rank.u64); + display_wl_with_final(if_num, wl_rank, rankx); + + // FIXME: does this help make the output a little easier + // to focus? + if (wl_print > 0) + debug("-----------\n"); + + } /* if (wl_loops > 1) */ +#endif /* HW_WL_MAJORITY */ + + // maybe print an error summary for the rank + if (wl_mask_err_rank != 0 || wl_val_err_rank != 0) { + debug("N%d.LMC%d.R%d: H/W Write-Leveling errors - %d bitmask, %d validity, %d retries, %d exhausted\n", + node, if_num, rankx, wl_mask_err_rank, + wl_val_err_rank, wloop_retries_total, + wloop_retries_exhausted); + } +} + +static void lmc_write_leveling(struct ddr_priv *priv) +{ + union cvmx_lmcx_config cfg; + int rankx; + char *s; + + /* + * 4.8.9 LMC Write Leveling + * + * LMC supports an automatic write leveling like that described in the + * JEDEC DDR3 specifications separately per byte-lane. + * + * All of DDR PLL, LMC CK, LMC DRESET, and early LMC initializations + * must be completed prior to starting this LMC write-leveling sequence. + * + * There are many possible procedures that will write-level all the + * attached DDR3 DRAM parts. One possibility is for software to simply + * write the desired values into LMC(0)_WLEVEL_RANK(0..3). This section + * describes one possible sequence that uses LMC's autowrite-leveling + * capabilities. + * + * 1. If the DQS/DQ delays on the board may be more than the ADD/CMD + * delays, then ensure that LMC(0)_CONFIG[EARLY_DQX] is set at this + * point. + * + * Do the remaining steps 2-7 separately for each rank i with attached + * DRAM. + * + * 2. Write LMC(0)_WLEVEL_RANKi = 0. + * + * 3. For x8 parts: + * + * Without changing any other fields in LMC(0)_WLEVEL_CTL, write + * LMC(0)_WLEVEL_CTL[LANEMASK] to select all byte lanes with attached + * DRAM. + * + * For x16 parts: + * + * Without changing any other fields in LMC(0)_WLEVEL_CTL, write + * LMC(0)_WLEVEL_CTL[LANEMASK] to select all even byte lanes with + * attached DRAM. + * + * 4. Without changing any other fields in LMC(0)_CONFIG, + * + * o write LMC(0)_SEQ_CTL[SEQ_SEL] to select write-leveling + * + * o write LMC(0)_CONFIG[RANKMASK] = (1 << i) + * + * o write LMC(0)_SEQ_CTL[INIT_START] = 1 + * + * LMC will initiate write-leveling at this point. Assuming + * LMC(0)_WLEVEL_CTL [SSET] = 0, LMC first enables write-leveling on + * the selected DRAM rank via a DDR3 MR1 write, then sequences + * through + * and accumulates write-leveling results for eight different delay + * settings twice, starting at a delay of zero in this case since + * LMC(0)_WLEVEL_RANKi[BYTE*<4:3>] = 0, increasing by 1/8 CK each + * setting, covering a total distance of one CK, then disables the + * write-leveling via another DDR3 MR1 write. + * + * After the sequence through 16 delay settings is complete: + * + * o LMC sets LMC(0)_WLEVEL_RANKi[STATUS] = 3 + * + * o LMC sets LMC(0)_WLEVEL_RANKi[BYTE*<2:0>] (for all ranks selected + * by LMC(0)_WLEVEL_CTL[LANEMASK]) to indicate the first write + * leveling result of 1 that followed result of 0 during the + * sequence, except that the LMC always writes + * LMC(0)_WLEVEL_RANKi[BYTE*<0>]=0. + * + * o Software can read the eight write-leveling results from the + * first pass through the delay settings by reading + * LMC(0)_WLEVEL_DBG[BITMASK] (after writing + * LMC(0)_WLEVEL_DBG[BYTE]). (LMC does not retain the writeleveling + * results from the second pass through the eight delay + * settings. They should often be identical to the + * LMC(0)_WLEVEL_DBG[BITMASK] results, though.) + * + * 5. Wait until LMC(0)_WLEVEL_RANKi[STATUS] != 2. + * + * LMC will have updated LMC(0)_WLEVEL_RANKi[BYTE*<2:0>] for all byte + * lanes selected by LMC(0)_WLEVEL_CTL[LANEMASK] at this point. + * LMC(0)_WLEVEL_RANKi[BYTE*<4:3>] will still be the value that + * software wrote in substep 2 above, which is 0. + * + * 6. For x16 parts: + * + * Without changing any other fields in LMC(0)_WLEVEL_CTL, write + * LMC(0)_WLEVEL_CTL[LANEMASK] to select all odd byte lanes with + * attached DRAM. + * + * Repeat substeps 4 and 5 with this new LMC(0)_WLEVEL_CTL[LANEMASK] + * setting. Skip to substep 7 if this has already been done. + * + * For x8 parts: + * + * Skip this substep. Go to substep 7. + * + * 7. Calculate LMC(0)_WLEVEL_RANKi[BYTE*<4:3>] settings for all byte + * lanes on all ranks with attached DRAM. + * + * At this point, all byte lanes on rank i with attached DRAM should + * have been write-leveled, and LMC(0)_WLEVEL_RANKi[BYTE*<2:0>] has + * the result for each byte lane. + * + * But note that the DDR3 write-leveling sequence will only determine + * the delay modulo the CK cycle time, and cannot determine how many + * additional CK cycles of delay are present. Software must calculate + * the number of CK cycles, or equivalently, the + * LMC(0)_WLEVEL_RANKi[BYTE*<4:3>] settings. + * + * This BYTE*<4:3> calculation is system/board specific. + * + * Many techniques can be used to calculate write-leveling BYTE*<4:3> + * values, including: + * + * o Known values for some byte lanes. + * + * o Relative values for some byte lanes relative to others. + * + * For example, suppose lane X is likely to require a larger + * write-leveling delay than lane Y. A BYTEX<2:0> value that is much + * smaller than the BYTEY<2:0> value may then indicate that the + * required lane X delay wrapped into the next CK, so BYTEX<4:3> + * should be set to BYTEY<4:3>+1. + * + * When ECC DRAM is not present (i.e. when DRAM is not attached to + * the DDR_CBS_0_* and DDR_CB<7:0> chip signals, or the + * DDR_DQS_<4>_* and DDR_DQ<35:32> chip signals), write + * LMC(0)_WLEVEL_RANK*[BYTE8] = LMC(0)_WLEVEL_RANK*[BYTE0], + * using the final calculated BYTE0 value. + * Write LMC(0)_WLEVEL_RANK*[BYTE4] = LMC(0)_WLEVEL_RANK*[BYTE0], + * using the final calculated BYTE0 value. + * + * 8. Initialize LMC(0)_WLEVEL_RANK* values for all unused ranks. + * + * Let rank i be a rank with attached DRAM. + * + * For all ranks j that do not have attached DRAM, set + * LMC(0)_WLEVEL_RANKj = LMC(0)_WLEVEL_RANKi. + */ + + rankx = 0; + wl_roundup = 0; + disable_hwl_validity = 0; + + // wl_pbm_pump: weight for write-leveling PBMs... + // 0 causes original behavior + // 1 allows a minority of 2 pbms to outscore a majority of 3 non-pbms + // 4 would allow a minority of 1 pbm to outscore a majority of 4 + // non-pbms + wl_pbm_pump = 4; // FIXME: is 4 too much? + + if (wl_loops) { + debug("N%d.LMC%d: Performing Hardware Write-Leveling\n", node, + if_num); + } else { + /* Force software write-leveling to run */ + wl_mask_err = 1; + debug("N%d.LMC%d: Forcing software Write-Leveling\n", node, + if_num); + } + + default_wl_rtt_nom = (ddr_type == DDR3_DRAM) ? + rttnom_20ohm : ddr4_rttnom_40ohm; + + cfg.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + ecc_ena = cfg.s.ecc_ena; + save_mode32b = cfg.cn78xx.mode32b; + cfg.cn78xx.mode32b = (!if_64b); + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), cfg.u64); + debug("%-45s : %d\n", "MODE32B", cfg.cn78xx.mode32b); + + s = lookup_env(priv, "ddr_wlevel_roundup"); + if (s) + wl_roundup = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_wlevel_printall"); + if (s) + wl_print = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_wlevel_pbm_bump"); + if (s) + wl_pbm_pump = strtoul(s, NULL, 0); + + // default to disable when RL sequential delay check is disabled + disable_hwl_validity = disable_sequential_delay_check; + s = lookup_env(priv, "ddr_disable_hwl_validity"); + if (s) + disable_hwl_validity = !!strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_wl_rtt_nom"); + if (s) + default_wl_rtt_nom = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_match_wl_rtt_nom"); + if (s) + match_wl_rtt_nom = !!simple_strtoul(s, NULL, 0); + + if (match_wl_rtt_nom) + mp1.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num)); + + // For DDR3, we do not touch WLEVEL_CTL fields OR_DIS or BITMASK + // For DDR4, we touch WLEVEL_CTL fields OR_DIS or BITMASK here + if (ddr_type == DDR4_DRAM) { + int default_or_dis = 1; + int default_bitmask = 0xff; + + // when x4, use only the lower nibble + if (dram_width == 4) { + default_bitmask = 0x0f; + if (wl_print) { + debug("N%d.LMC%d: WLEVEL_CTL: default bitmask is 0x%02x for DDR4 x4\n", + node, if_num, default_bitmask); + } + } + + wl_ctl.u64 = lmc_rd(priv, CVMX_LMCX_WLEVEL_CTL(if_num)); + wl_ctl.s.or_dis = default_or_dis; + wl_ctl.s.bitmask = default_bitmask; + + // allow overrides + s = lookup_env(priv, "ddr_wlevel_ctl_or_dis"); + if (s) + wl_ctl.s.or_dis = !!strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_wlevel_ctl_bitmask"); + if (s) + wl_ctl.s.bitmask = simple_strtoul(s, NULL, 0); + + // print only if not defaults + if (wl_ctl.s.or_dis != default_or_dis || + wl_ctl.s.bitmask != default_bitmask) { + debug("N%d.LMC%d: WLEVEL_CTL: or_dis=%d, bitmask=0x%02x\n", + node, if_num, wl_ctl.s.or_dis, wl_ctl.s.bitmask); + } + + // always write + lmc_wr(priv, CVMX_LMCX_WLEVEL_CTL(if_num), wl_ctl.u64); + } + + // Start the hardware write-leveling loop per rank + for (rankx = 0; rankx < dimm_count * 4; rankx++) + lmc_write_leveling_loop(priv, rankx); + + cfg.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + cfg.cn78xx.mode32b = save_mode32b; + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), cfg.u64); + debug("%-45s : %d\n", "MODE32B", cfg.cn78xx.mode32b); + + // At the end of HW Write Leveling, check on some DESKEW things... + if (!disable_deskew_training) { + struct deskew_counts dsk_counts; + int retry_count = 0; + + debug("N%d.LMC%d: Check Deskew Settings before Read-Leveling.\n", + node, if_num); + + do { + validate_deskew_training(priv, rank_mask, if_num, + &dsk_counts, 1); + + // only RAWCARD A or B will not benefit from + // retraining if there's only saturation + // or any rawcard if there is a nibble error + if ((!spd_rawcard_aorb && dsk_counts.saturated > 0) || + (dsk_counts.nibrng_errs != 0 || + dsk_counts.nibunl_errs != 0)) { + retry_count++; + debug("N%d.LMC%d: Deskew Status indicates saturation or nibble errors - retry %d Training.\n", + node, if_num, retry_count); + perform_deskew_training(priv, rank_mask, if_num, + spd_rawcard_aorb); + } else { + break; + } + } while (retry_count < 5); + } +} + +static void lmc_workaround(struct ddr_priv *priv) +{ + /* Workaround Trcd overflow by using Additive latency. */ + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) { + union cvmx_lmcx_modereg_params0 mp0; + union cvmx_lmcx_timing_params1 tp1; + union cvmx_lmcx_control ctrl; + int rankx; + + tp1.u64 = lmc_rd(priv, CVMX_LMCX_TIMING_PARAMS1(if_num)); + mp0.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num)); + ctrl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + + if (tp1.cn78xx.trcd == 0) { + debug("Workaround Trcd overflow by using Additive latency.\n"); + /* Hard code this to 12 and enable additive latency */ + tp1.cn78xx.trcd = 12; + mp0.s.al = 2; /* CL-2 */ + ctrl.s.pocas = 1; + + debug("MODEREG_PARAMS0 : 0x%016llx\n", + mp0.u64); + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num), + mp0.u64); + debug("TIMING_PARAMS1 : 0x%016llx\n", + tp1.u64); + lmc_wr(priv, CVMX_LMCX_TIMING_PARAMS1(if_num), tp1.u64); + + debug("LMC_CONTROL : 0x%016llx\n", + ctrl.u64); + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctrl.u64); + + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + /* MR1 */ + ddr4_mrw(priv, if_num, rankx, -1, 1, 0); + } + } + } + + // this is here just for output, to allow check of the Deskew + // settings one last time... + if (!disable_deskew_training) { + struct deskew_counts dsk_counts; + + debug("N%d.LMC%d: Check Deskew Settings before software Write-Leveling.\n", + node, if_num); + validate_deskew_training(priv, rank_mask, if_num, &dsk_counts, + 3); + } + + /* + * Workaround Errata 26304 (T88@2.0, O75@1.x, O78@2.x) + * + * When the CSRs LMCX_DLL_CTL3[WR_DESKEW_ENA] = 1 AND + * LMCX_PHY_CTL2[DQS[0..8]_DSK_ADJ] > 4, set + * LMCX_EXT_CONFIG[DRIVE_ENA_BPRCH] = 1. + */ + if (octeon_is_cpuid(OCTEON_CN78XX_PASS2_X) || + octeon_is_cpuid(OCTEON_CNF75XX_PASS1_X)) { + union cvmx_lmcx_dll_ctl3 dll_ctl3; + union cvmx_lmcx_phy_ctl2 phy_ctl2; + union cvmx_lmcx_ext_config ext_cfg; + int increased_dsk_adj = 0; + int byte; + + phy_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL2(if_num)); + ext_cfg.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num)); + dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); + + for (byte = 0; byte < 8; ++byte) { + if (!(if_bytemask & (1 << byte))) + continue; + increased_dsk_adj |= + (((phy_ctl2.u64 >> (byte * 3)) & 0x7) > 4); + } + + if (dll_ctl3.s.wr_deskew_ena == 1 && increased_dsk_adj) { + ext_cfg.s.drive_ena_bprch = 1; + lmc_wr(priv, CVMX_LMCX_EXT_CONFIG(if_num), ext_cfg.u64); + debug("LMC%d: Forcing DRIVE_ENA_BPRCH for Workaround Errata 26304.\n", + if_num); + } + } +} + +// Software Write-Leveling block + +#define VREF_RANGE1_LIMIT 0x33 // range1 is valid for 0x00 - 0x32 +#define VREF_RANGE2_LIMIT 0x18 // range2 is valid for 0x00 - 0x17 +// full window is valid for 0x00 to 0x4A +// let 0x00 - 0x17 be range2, 0x18 - 0x4a be range 1 +#define VREF_LIMIT (VREF_RANGE1_LIMIT + VREF_RANGE2_LIMIT) +#define VREF_FINAL (VREF_LIMIT - 1) + +enum sw_wl_status { + WL_ESTIMATED = 0, /* HW/SW wleveling failed. Reslt estimated */ + WL_HARDWARE = 1, /* H/W wleveling succeeded */ + WL_SOFTWARE = 2, /* S/W wleveling passed 2 contiguous setting */ + WL_SOFTWARE1 = 3, /* S/W wleveling passed 1 marginal setting */ +}; + +static u64 rank_addr __section(".data"); +static int vref_val __section(".data"); +static int final_vref_val __section(".data"); +static int final_vref_range __section(".data"); +static int start_vref_val __section(".data"); +static int computed_final_vref_val __section(".data"); +static char best_vref_val_count __section(".data"); +static char vref_val_count __section(".data"); +static char best_vref_val_start __section(".data"); +static char vref_val_start __section(".data"); +static int bytes_failed __section(".data"); +static enum sw_wl_status byte_test_status[9] __section(".data"); +static enum sw_wl_status sw_wl_rank_status __section(".data"); +static int sw_wl_failed __section(".data"); +static int sw_wl_hw __section(".data"); +static int measured_vref_flag __section(".data"); + +static void ddr4_vref_loop(struct ddr_priv *priv, int rankx) +{ + char *s; + + if (vref_val < VREF_FINAL) { + int vrange, vvalue; + + if (vref_val < VREF_RANGE2_LIMIT) { + vrange = 1; + vvalue = vref_val; + } else { + vrange = 0; + vvalue = vref_val - VREF_RANGE2_LIMIT; + } + + set_vref(priv, if_num, rankx, vrange, vvalue); + } else { /* if (vref_val < VREF_FINAL) */ + /* Print the final vref value first. */ + + /* Always print the computed first if its valid */ + if (computed_final_vref_val >= 0) { + debug("N%d.LMC%d.R%d: vref Computed Summary : %2d (0x%02x)\n", + node, if_num, rankx, + computed_final_vref_val, computed_final_vref_val); + } + + if (!measured_vref_flag) { // setup to use the computed + best_vref_val_count = 1; + final_vref_val = computed_final_vref_val; + } else { // setup to use the measured + if (best_vref_val_count > 0) { + best_vref_val_count = + max(best_vref_val_count, (char)2); + final_vref_val = best_vref_val_start + + divide_nint(best_vref_val_count - 1, 2); + + if (final_vref_val < VREF_RANGE2_LIMIT) { + final_vref_range = 1; + } else { + final_vref_range = 0; + final_vref_val -= VREF_RANGE2_LIMIT; + } + + int vvlo = best_vref_val_start; + int vrlo; + int vvhi = best_vref_val_start + + best_vref_val_count - 1; + int vrhi; + + if (vvlo < VREF_RANGE2_LIMIT) { + vrlo = 2; + } else { + vrlo = 1; + vvlo -= VREF_RANGE2_LIMIT; + } + + if (vvhi < VREF_RANGE2_LIMIT) { + vrhi = 2; + } else { + vrhi = 1; + vvhi -= VREF_RANGE2_LIMIT; + } + debug("N%d.LMC%d.R%d: vref Training Summary : 0x%02x/%1d <----- 0x%02x/%1d -----> 0x%02x/%1d, range: %2d\n", + node, if_num, rankx, vvlo, vrlo, + final_vref_val, + final_vref_range + 1, vvhi, vrhi, + best_vref_val_count - 1); + + } else { + /* + * If nothing passed use the default vref + * value for this rank + */ + union cvmx_lmcx_modereg_params2 mp2; + + mp2.u64 = + lmc_rd(priv, + CVMX_LMCX_MODEREG_PARAMS2(if_num)); + final_vref_val = (mp2.u64 >> + (rankx * 10 + 3)) & 0x3f; + final_vref_range = (mp2.u64 >> + (rankx * 10 + 9)) & 0x01; + + debug("N%d.LMC%d.R%d: vref Using Default : %2d <----- %2d (0x%02x) -----> %2d, range%1d\n", + node, if_num, rankx, final_vref_val, + final_vref_val, final_vref_val, + final_vref_val, final_vref_range + 1); + } + } + + // allow override + s = lookup_env(priv, "ddr%d_vref_val_%1d%1d", + if_num, !!(rankx & 2), !!(rankx & 1)); + if (s) + final_vref_val = strtoul(s, NULL, 0); + + set_vref(priv, if_num, rankx, final_vref_range, final_vref_val); + } +} + +#define WL_MIN_NO_ERRORS_COUNT 3 // FIXME? three passes without errors + +static int errors __section(".data"); +static int byte_delay[9] __section(".data"); +static u64 bytemask __section(".data"); +static int bytes_todo __section(".data"); +static int no_errors_count __section(".data"); +static u64 bad_bits[2] __section(".data"); +#if ENABLE_SW_WLEVEL_UTILIZATION +static u64 sum_dram_dclk __section(".data"); +static u64 sum_dram_ops __section(".data"); +static u64 start_dram_dclk __section(".data"); +static u64 stop_dram_dclk __section(".data"); +static u64 start_dram_ops __section(".data"); +static u64 stop_dram_ops __section(".data"); +#endif + +static void lmc_sw_write_leveling_loop(struct ddr_priv *priv, int rankx) +{ + int delay; + int b; + + // write the current set of WL delays + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), wl_rank.u64); + wl_rank.u64 = lmc_rd(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num)); + + // do the test + if (sw_wl_hw) { + errors = run_best_hw_patterns(priv, if_num, rank_addr, + DBTRAIN_TEST, bad_bits); + errors &= bytes_todo; // keep only the ones we are still doing + } else { +#if ENABLE_SW_WLEVEL_UTILIZATION + start_dram_dclk = lmc_rd(priv, CVMX_LMCX_DCLK_CNT(if_num)); + start_dram_ops = lmc_rd(priv, CVMX_LMCX_OPS_CNT(if_num)); +#endif + errors = test_dram_byte64(priv, if_num, rank_addr, bytemask, + bad_bits); + +#if ENABLE_SW_WLEVEL_UTILIZATION + stop_dram_dclk = lmc_rd(priv, CVMX_LMCX_DCLK_CNT(if_num)); + stop_dram_ops = lmc_rd(priv, CVMX_LMCX_OPS_CNT(if_num)); + sum_dram_dclk += stop_dram_dclk - start_dram_dclk; + sum_dram_ops += stop_dram_ops - start_dram_ops; +#endif + } + + debug("WL pass1: test_dram_byte returned 0x%x\n", errors); + + // remember, errors will not be returned for byte-lanes that have + // maxxed out... + if (errors == 0) { + no_errors_count++; // bump + // bypass check/update completely + if (no_errors_count > 1) + return; // to end of do-while + } else { + no_errors_count = 0; // reset + } + + // check errors by byte + for (b = 0; b < 9; ++b) { + if (!(bytes_todo & (1 << b))) + continue; + + delay = byte_delay[b]; + // yes, an error in this byte lane + if (errors & (1 << b)) { + debug(" byte %d delay %2d Errors\n", b, delay); + // since this byte had an error, we move to the next + // delay value, unless done with it + delay += 8; // incr by 8 to do delay high-order bits + if (delay < 32) { + upd_wl_rank(&wl_rank, b, delay); + debug(" byte %d delay %2d New\n", + b, delay); + byte_delay[b] = delay; + } else { + // reached max delay, maybe really done with + // this byte +#if SWL_TRY_HWL_ALT + // consider an alt only for computed VREF and + if (!measured_vref_flag && + (hwl_alts[rankx].hwl_alt_mask & (1 << b))) { + // if an alt exists... + // just orig low-3 bits + int bad_delay = delay & 0x6; + + // yes, use it + delay = hwl_alts[rankx].hwl_alt_delay[b]; + // clear that flag + hwl_alts[rankx].hwl_alt_mask &= + ~(1 << b); + upd_wl_rank(&wl_rank, b, delay); + byte_delay[b] = delay; + debug(" byte %d delay %2d ALTERNATE\n", + b, delay); + debug("N%d.LMC%d.R%d: SWL: Byte %d: %d FAIL, trying ALTERNATE %d\n", + node, if_num, + rankx, b, bad_delay, delay); + + } else +#endif /* SWL_TRY_HWL_ALT */ + { + unsigned int bits_bad; + + if (b < 8) { + // test no longer, remove from + // byte mask + bytemask &= + ~(0xffULL << (8 * b)); + bits_bad = (unsigned int) + ((bad_bits[0] >> + (8 * b)) & 0xffUL); + } else { + bits_bad = (unsigned int) + (bad_bits[1] & 0xffUL); + } + + // remove from bytes to do + bytes_todo &= ~(1 << b); + // make sure this is set for this case + byte_test_status[b] = WL_ESTIMATED; + debug(" byte %d delay %2d Exhausted\n", + b, delay); + if (!measured_vref_flag) { + // this is too noisy when doing + // measured VREF + debug("N%d.LMC%d.R%d: SWL: Byte %d (0x%02x): delay %d EXHAUSTED\n", + node, if_num, rankx, + b, bits_bad, delay); + } + } + } + } else { + // no error, stay with current delay, but keep testing + // it... + debug(" byte %d delay %2d Passed\n", b, delay); + byte_test_status[b] = WL_HARDWARE; // change status + } + } /* for (b = 0; b < 9; ++b) */ +} + +static void sw_write_lvl_use_ecc(struct ddr_priv *priv, int rankx) +{ + int save_byte8 = wl_rank.s.byte8; + + byte_test_status[8] = WL_HARDWARE; /* H/W delay value */ + + if (save_byte8 != wl_rank.s.byte3 && + save_byte8 != wl_rank.s.byte4) { + int test_byte8 = save_byte8; + int test_byte8_error; + int byte8_error = 0x1f; + int adder; + int avg_bytes = divide_nint(wl_rank.s.byte3 + wl_rank.s.byte4, + 2); + + for (adder = 0; adder <= 32; adder += 8) { + test_byte8_error = abs((adder + save_byte8) - + avg_bytes); + if (test_byte8_error < byte8_error) { + byte8_error = test_byte8_error; + test_byte8 = save_byte8 + adder; + } + } + +#if SW_WL_CHECK_PATCH + // only do the check if we are not using measured VREF + if (!measured_vref_flag) { + /* Use only even settings, rounding down... */ + test_byte8 &= ~1; + + // do validity check on the calculated ECC delay value + // this depends on the DIMM type + if (spd_rdimm) { // RDIMM + // but not mini-RDIMM + if (spd_dimm_type != 5) { + // it can be > byte4, but should never + // be > byte3 + if (test_byte8 > wl_rank.s.byte3) { + /* say it is still estimated */ + byte_test_status[8] = + WL_ESTIMATED; + } + } + } else { // UDIMM + if (test_byte8 < wl_rank.s.byte3 || + test_byte8 > wl_rank.s.byte4) { + // should never be outside the + // byte 3-4 range + /* say it is still estimated */ + byte_test_status[8] = WL_ESTIMATED; + } + } + /* + * Report whenever the calculation appears bad. + * This happens if some of the original values were off, + * or unexpected geometry from DIMM type, or custom + * circuitry (NIC225E, I am looking at you!). + * We will trust the calculated value, and depend on + * later testing to catch any instances when that + * value is truly bad. + */ + // ESTIMATED means there may be an issue + if (byte_test_status[8] == WL_ESTIMATED) { + debug("N%d.LMC%d.R%d: SWL: (%cDIMM): calculated ECC delay unexpected (%d/%d/%d)\n", + node, if_num, rankx, + (spd_rdimm ? 'R' : 'U'), wl_rank.s.byte4, + test_byte8, wl_rank.s.byte3); + byte_test_status[8] = WL_HARDWARE; + } + } +#endif /* SW_WL_CHECK_PATCH */ + /* Use only even settings */ + wl_rank.s.byte8 = test_byte8 & ~1; + } + + if (wl_rank.s.byte8 != save_byte8) { + /* Change the status if s/w adjusted the delay */ + byte_test_status[8] = WL_SOFTWARE; /* Estimated delay */ + } +} + +static __maybe_unused void parallel_wl_block_delay(struct ddr_priv *priv, + int rankx) +{ + int errors; + int byte_delay[8]; + int byte_passed[8]; + u64 bytemask; + u64 bitmask; + int wl_offset; + int bytes_todo; + int sw_wl_offset = 1; + int delay; + int b; + + for (b = 0; b < 8; ++b) + byte_passed[b] = 0; + + bytes_todo = if_bytemask; + + for (wl_offset = sw_wl_offset; wl_offset >= 0; --wl_offset) { + debug("Starting wl_offset for-loop: %d\n", wl_offset); + + bytemask = 0; + + for (b = 0; b < 8; ++b) { + byte_delay[b] = 0; + // this does not contain fully passed bytes + if (!(bytes_todo & (1 << b))) + continue; + + // reset across passes if not fully passed + byte_passed[b] = 0; + upd_wl_rank(&wl_rank, b, 0); // all delays start at 0 + bitmask = ((!if_64b) && (b == 4)) ? 0x0f : 0xff; + // set the bytes bits in the bytemask + bytemask |= bitmask << (8 * b); + } /* for (b = 0; b < 8; ++b) */ + + // start a pass if there is any byte lane to test + while (bytemask != 0) { + debug("Starting bytemask while-loop: 0x%llx\n", + bytemask); + + // write this set of WL delays + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), + wl_rank.u64); + wl_rank.u64 = lmc_rd(priv, + CVMX_LMCX_WLEVEL_RANKX(rankx, + if_num)); + + // do the test + if (sw_wl_hw) { + errors = run_best_hw_patterns(priv, if_num, + rank_addr, + DBTRAIN_TEST, + NULL) & 0xff; + } else { + errors = test_dram_byte64(priv, if_num, + rank_addr, bytemask, + NULL); + } + + debug("test_dram_byte returned 0x%x\n", errors); + + // check errors by byte + for (b = 0; b < 8; ++b) { + if (!(bytes_todo & (1 << b))) + continue; + + delay = byte_delay[b]; + if (errors & (1 << b)) { // yes, an error + debug(" byte %d delay %2d Errors\n", + b, delay); + byte_passed[b] = 0; + } else { // no error + byte_passed[b] += 1; + // Look for consecutive working settings + if (byte_passed[b] == (1 + wl_offset)) { + debug(" byte %d delay %2d FULLY Passed\n", + b, delay); + if (wl_offset == 1) { + byte_test_status[b] = + WL_SOFTWARE; + } else if (wl_offset == 0) { + byte_test_status[b] = + WL_SOFTWARE1; + } + + // test no longer, remove + // from byte mask this pass + bytemask &= ~(0xffULL << + (8 * b)); + // remove completely from + // concern + bytes_todo &= ~(1 << b); + // on to the next byte, bypass + // delay updating!! + continue; + } else { + debug(" byte %d delay %2d Passed\n", + b, delay); + } + } + + // error or no, here we move to the next delay + // value for this byte, unless done all delays + // only a byte that has "fully passed" will + // bypass around this, + delay += 2; + if (delay < 32) { + upd_wl_rank(&wl_rank, b, delay); + debug(" byte %d delay %2d New\n", + b, delay); + byte_delay[b] = delay; + } else { + // reached max delay, done with this + // byte + debug(" byte %d delay %2d Exhausted\n", + b, delay); + // test no longer, remove from byte + // mask this pass + bytemask &= ~(0xffULL << (8 * b)); + } + } /* for (b = 0; b < 8; ++b) */ + debug("End of for-loop: bytemask 0x%llx\n", bytemask); + } /* while (bytemask != 0) */ + } + + for (b = 0; b < 8; ++b) { + // any bytes left in bytes_todo did not pass + if (bytes_todo & (1 << b)) { + union cvmx_lmcx_rlevel_rankx lmc_rlevel_rank; + + /* + * Last resort. Use Rlevel settings to estimate + * Wlevel if software write-leveling fails + */ + debug("Using RLEVEL as WLEVEL estimate for byte %d\n", + b); + lmc_rlevel_rank.u64 = + lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + rlevel_to_wlevel(&lmc_rlevel_rank, &wl_rank, b); + } + } /* for (b = 0; b < 8; ++b) */ +} + +static int lmc_sw_write_leveling(struct ddr_priv *priv) +{ + /* Try to determine/optimize write-level delays experimentally. */ + union cvmx_lmcx_wlevel_rankx wl_rank_hw_res; + union cvmx_lmcx_config cfg; + int rankx; + int byte; + char *s; + int i; + + int active_rank; + int sw_wl_enable = 1; /* FIX... Should be customizable. */ + int interfaces; + + static const char * const wl_status_strings[] = { + "(e)", + " ", + " ", + "(1)" + }; + + // FIXME: make HW-assist the default now? + int sw_wl_hw_default = SW_WLEVEL_HW_DEFAULT; + int dram_connection = c_cfg->dram_connection; + + s = lookup_env(priv, "ddr_sw_wlevel_hw"); + if (s) + sw_wl_hw_default = !!strtoul(s, NULL, 0); + if (!if_64b) // must use SW algo if 32-bit mode + sw_wl_hw_default = 0; + + // can never use hw-assist + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) + sw_wl_hw_default = 0; + + s = lookup_env(priv, "ddr_software_wlevel"); + if (s) + sw_wl_enable = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr%d_dram_connection", if_num); + if (s) + dram_connection = !!strtoul(s, NULL, 0); + + cvmx_rng_enable(); + +#if SWL_WITH_HW_ALTS_CHOOSE_SW + // Choose the SW algo for SWL if any HWL alternates were found + // NOTE: we have to do this here, and for all, since HW-assist + // including ECC requires ECC enable + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!sw_wl_enable) + break; + if (!(rank_mask & (1 << rankx))) + continue; + + // if we are doing HW-assist, and there are alternates, + // switch to SW-algorithm for all + if (sw_wl_hw_default && hwl_alts[rankx].hwl_alt_mask) { + debug("N%d.LMC%d.R%d: Using SW algorithm for write-leveling this rank\n", + node, if_num, rankx); + sw_wl_hw_default = 0; + break; + } + } +#endif + + /* + * Get the measured_vref setting from the config, check for an + * override... + */ + /* NOTE: measured_vref=1 (ON) means force use of MEASURED vref... */ + // NOTE: measured VREF can only be done for DDR4 + if (ddr_type == DDR4_DRAM) { + measured_vref_flag = c_cfg->measured_vref; + s = lookup_env(priv, "ddr_measured_vref"); + if (s) + measured_vref_flag = !!strtoul(s, NULL, 0); + } else { + measured_vref_flag = 0; // OFF for DDR3 + } + + /* + * Ensure disabled ECC for DRAM tests using the SW algo, else leave + * it untouched + */ + if (!sw_wl_hw_default) { + cfg.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + cfg.cn78xx.ecc_ena = 0; + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), cfg.u64); + } + + /* + * We need to track absolute rank number, as well as how many + * active ranks we have. Two single rank DIMMs show up as + * ranks 0 and 2, but only 2 ranks are active. + */ + active_rank = 0; + + interfaces = __builtin_popcount(if_mask); + + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + final_vref_range = 0; + start_vref_val = 0; + computed_final_vref_val = -1; + sw_wl_rank_status = WL_HARDWARE; + sw_wl_failed = 0; + sw_wl_hw = sw_wl_hw_default; + + if (!sw_wl_enable) + break; + + if (!(rank_mask & (1 << rankx))) + continue; + + debug("N%d.LMC%d.R%d: Performing Software Write-Leveling %s\n", + node, if_num, rankx, + (sw_wl_hw) ? "with H/W assist" : + "with S/W algorithm"); + + if (ddr_type == DDR4_DRAM && num_ranks != 4) { + // always compute when we can... + computed_final_vref_val = + compute_vref_val(priv, if_num, rankx, dimm_count, + num_ranks, imp_val, + is_stacked_die, dram_connection); + + // but only use it if allowed + if (!measured_vref_flag) { + // skip all the measured vref processing, + // just the final setting + start_vref_val = VREF_FINAL; + } + } + + /* Save off the h/w wl results */ + wl_rank_hw_res.u64 = lmc_rd(priv, + CVMX_LMCX_WLEVEL_RANKX(rankx, + if_num)); + + vref_val_count = 0; + vref_val_start = 0; + best_vref_val_count = 0; + best_vref_val_start = 0; + + /* Loop one extra time using the Final vref value. */ + for (vref_val = start_vref_val; vref_val < VREF_LIMIT; + ++vref_val) { + if (ddr_type == DDR4_DRAM) + ddr4_vref_loop(priv, rankx); + + /* Restore the saved value */ + wl_rank.u64 = wl_rank_hw_res.u64; + + for (byte = 0; byte < 9; ++byte) + byte_test_status[byte] = WL_ESTIMATED; + + if (wl_mask_err == 0) { + /* + * Determine address of DRAM to test for + * pass 1 of software write leveling. + */ + rank_addr = active_rank * + (1ull << (pbank_lsb - bunk_enable + + (interfaces / 2))); + + /* + * Adjust address for boot bus hole in memory + * map. + */ + if (rank_addr > 0x10000000) + rank_addr += 0x10000000; + + debug("N%d.LMC%d.R%d: Active Rank %d Address: 0x%llx\n", + node, if_num, rankx, active_rank, + rank_addr); + + // start parallel write-leveling block for + // delay high-order bits + errors = 0; + no_errors_count = 0; +#if ENABLE_SW_WLEVEL_UTILIZATION + sum_dram_dclk = 0; + sum_dram_ops = 0; +#endif + + if (if_64b) { + bytes_todo = (sw_wl_hw) ? + if_bytemask : 0xFF; + bytemask = ~0ULL; + } else { + // 32-bit, must be using SW algo, + // only data bytes + bytes_todo = 0x0f; + bytemask = 0x00000000ffffffffULL; + } + + for (byte = 0; byte < 9; ++byte) { + if (!(bytes_todo & (1 << byte))) { + byte_delay[byte] = 0; + } else { + byte_delay[byte] = + get_wl_rank(&wl_rank, byte); + } + } /* for (byte = 0; byte < 9; ++byte) */ + + do { + lmc_sw_write_leveling_loop(priv, rankx); + } while (no_errors_count < + WL_MIN_NO_ERRORS_COUNT); + +#if ENABLE_SW_WLEVEL_UTILIZATION + if (!sw_wl_hw) { + u64 percent_x10; + + if (sum_dram_dclk == 0) + sum_dram_dclk = 1; + percent_x10 = sum_dram_ops * 1000 / + sum_dram_dclk; + debug("N%d.LMC%d.R%d: ops %llu, cycles %llu, used %llu.%llu%%\n", + node, if_num, rankx, sum_dram_ops, + sum_dram_dclk, percent_x10 / 10, + percent_x10 % 10); + } +#endif + if (errors) { + debug("End WLEV_64 while loop: vref_val %d(0x%x), errors 0x%02x\n", + vref_val, vref_val, errors); + } + // end parallel write-leveling block for + // delay high-order bits + + // if we used HW-assist, we did the ECC byte + // when approp. + if (sw_wl_hw) { + if (wl_print) { + debug("N%d.LMC%d.R%d: HW-assisted SWL - ECC estimate not needed.\n", + node, if_num, rankx); + } + goto no_ecc_estimate; + } + + if ((if_bytemask & 0xff) == 0xff) { + if (use_ecc) { + sw_write_lvl_use_ecc(priv, + rankx); + } else { + /* H/W delay value */ + byte_test_status[8] = + WL_HARDWARE; + /* ECC is not used */ + wl_rank.s.byte8 = + wl_rank.s.byte0; + } + } else { + if (use_ecc) { + /* Estimate the ECC byte dly */ + // add hi-order to b4 + wl_rank.s.byte4 |= + (wl_rank.s.byte3 & + 0x38); + if ((wl_rank.s.byte4 & 0x06) < + (wl_rank.s.byte3 & 0x06)) { + // must be next clock + wl_rank.s.byte4 += 8; + } + } else { + /* ECC is not used */ + wl_rank.s.byte4 = + wl_rank.s.byte0; + } + + /* + * Change the status if s/w adjusted + * the delay + */ + /* Estimated delay */ + byte_test_status[4] = WL_SOFTWARE; + } /* if ((if_bytemask & 0xff) == 0xff) */ + } /* if (wl_mask_err == 0) */ + +no_ecc_estimate: + + bytes_failed = 0; + for (byte = 0; byte < 9; ++byte) { + /* Don't accumulate errors for untested bytes */ + if (!(if_bytemask & (1 << byte))) + continue; + bytes_failed += + (byte_test_status[byte] == WL_ESTIMATED); + } + + /* vref training loop is only used for DDR4 */ + if (ddr_type != DDR4_DRAM) + break; + + if (bytes_failed == 0) { + if (vref_val_count == 0) + vref_val_start = vref_val; + + ++vref_val_count; + if (vref_val_count > best_vref_val_count) { + best_vref_val_count = vref_val_count; + best_vref_val_start = vref_val_start; + debug("N%d.LMC%d.R%d: vref Training (%2d) : 0x%02x <----- ???? -----> 0x%02x\n", + node, if_num, rankx, vref_val, + best_vref_val_start, + best_vref_val_start + + best_vref_val_count - 1); + } + } else { + vref_val_count = 0; + debug("N%d.LMC%d.R%d: vref Training (%2d) : failed\n", + node, if_num, rankx, vref_val); + } + } + + /* + * Determine address of DRAM to test for software write + * leveling. + */ + rank_addr = active_rank * (1ull << (pbank_lsb - bunk_enable + + (interfaces / 2))); + /* Adjust address for boot bus hole in memory map. */ + if (rank_addr > 0x10000000) + rank_addr += 0x10000000; + + debug("Rank Address: 0x%llx\n", rank_addr); + + if (bytes_failed) { +#if !DISABLE_SW_WL_PASS_2 + debug("N%d.LMC%d.R%d: Starting SW Write-leveling pass 2\n", + node, if_num, rankx); + + sw_wl_rank_status = WL_SOFTWARE; + + /* + * If s/w fixups failed then retry using s/w + * write-leveling. + */ + if (wl_mask_err == 0) { + /* + * h/w succeeded but s/w fixups failed. + * So retry s/w. + */ + debug("N%d.LMC%d.R%d: Retrying software Write-Leveling.\n", + node, if_num, rankx); + } + + // start parallel write-leveling block for delay + // low-order bits + parallel_wl_block_delay(priv, rankx); + + if (use_ecc) { + /* + * ECC byte has to be estimated. Take the + * average of the two surrounding bytes. + */ + int test_byte8 = divide_nint(wl_rank.s.byte3 + + wl_rank.s.byte4 + + 2, 2); + /* Use only even settings */ + wl_rank.s.byte8 = test_byte8 & ~1; + /* Estimated delay */ + byte_test_status[8] = WL_ESTIMATED; + } else { + /* H/W delay value */ + byte_test_status[8] = WL_HARDWARE; + /* ECC is not used */ + wl_rank.s.byte8 = wl_rank.s.byte0; + } + + /* Set delays for unused bytes to match byte 0. */ + for (byte = 0; byte < 8; ++byte) { + if ((if_bytemask & (1 << byte))) + continue; + upd_wl_rank(&wl_rank, byte, wl_rank.s.byte0); + byte_test_status[byte] = WL_SOFTWARE; + } +#else /* !DISABLE_SW_WL_PASS_2 */ + // FIXME? the big hammer, did not even try SW WL pass2, + // assume only chip reset will help + debug("N%d.LMC%d.R%d: S/W write-leveling pass 1 failed\n", + node, if_num, rankx); + sw_wl_failed = 1; +#endif /* !DISABLE_SW_WL_PASS_2 */ + } else { /* if (bytes_failed) */ + // SW WL pass 1 was OK, write the settings + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), + wl_rank.u64); + wl_rank.u64 = lmc_rd(priv, + CVMX_LMCX_WLEVEL_RANKX(rankx, + if_num)); + +#if SW_WL_CHECK_PATCH + // do validity check on the delay values by running + // the test 1 more time... + // FIXME: we really need to check the ECC byte setting + // here as well, so we need to enable ECC for this test! + // if there are any errors, claim SW WL failure + u64 datamask = (if_64b) ? 0xffffffffffffffffULL : + 0x00000000ffffffffULL; + int errors; + + // do the test + if (sw_wl_hw) { + errors = run_best_hw_patterns(priv, if_num, + rank_addr, + DBTRAIN_TEST, + NULL) & 0xff; + } else { + errors = test_dram_byte64(priv, if_num, + rank_addr, datamask, + NULL); + } + + if (errors) { + debug("N%d.LMC%d.R%d: Wlevel Rank Final Test errors 0x%03x\n", + node, if_num, rankx, errors); + sw_wl_failed = 1; + } +#endif /* SW_WL_CHECK_PATCH */ + } /* if (bytes_failed) */ + + // FIXME? dump the WL settings, so we get more of a clue + // as to what happened where + debug("N%d.LMC%d.R%d: Wlevel Rank %#4x, 0x%016llX : %2d%3s %2d%3s %2d%3s %2d%3s %2d%3s %2d%3s %2d%3s %2d%3s %2d%3s %s\n", + node, if_num, rankx, wl_rank.s.status, wl_rank.u64, + wl_rank.s.byte8, wl_status_strings[byte_test_status[8]], + wl_rank.s.byte7, wl_status_strings[byte_test_status[7]], + wl_rank.s.byte6, wl_status_strings[byte_test_status[6]], + wl_rank.s.byte5, wl_status_strings[byte_test_status[5]], + wl_rank.s.byte4, wl_status_strings[byte_test_status[4]], + wl_rank.s.byte3, wl_status_strings[byte_test_status[3]], + wl_rank.s.byte2, wl_status_strings[byte_test_status[2]], + wl_rank.s.byte1, wl_status_strings[byte_test_status[1]], + wl_rank.s.byte0, wl_status_strings[byte_test_status[0]], + (sw_wl_rank_status == WL_HARDWARE) ? "" : "(s)"); + + // finally, check for fatal conditions: either chip reset + // right here, or return error flag + if ((ddr_type == DDR4_DRAM && best_vref_val_count == 0) || + sw_wl_failed) { + if (!ddr_disable_chip_reset) { // do chip RESET + printf("N%d.LMC%d.R%d: INFO: Short memory test indicates a retry is needed. Resetting node...\n", + node, if_num, rankx); + mdelay(500); + do_reset(NULL, 0, 0, NULL); + } else { + // return error flag so LMC init can be retried. + debug("N%d.LMC%d.R%d: INFO: Short memory test indicates a retry is needed. Restarting LMC init...\n", + node, if_num, rankx); + return -EAGAIN; // 0 indicates restart possible. + } + } + active_rank++; + } + + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + int parameter_set = 0; + u64 value; + + if (!(rank_mask & (1 << rankx))) + continue; + + wl_rank.u64 = lmc_rd(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, + if_num)); + + for (i = 0; i < 9; ++i) { + s = lookup_env(priv, "ddr%d_wlevel_rank%d_byte%d", + if_num, rankx, i); + if (s) { + parameter_set |= 1; + value = strtoul(s, NULL, 0); + + upd_wl_rank(&wl_rank, i, value); + } + } + + s = lookup_env_ull(priv, "ddr%d_wlevel_rank%d", if_num, rankx); + if (s) { + parameter_set |= 1; + value = strtoull(s, NULL, 0); + wl_rank.u64 = value; + } + + if (parameter_set) { + lmc_wr(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num), + wl_rank.u64); + wl_rank.u64 = + lmc_rd(priv, CVMX_LMCX_WLEVEL_RANKX(rankx, if_num)); + display_wl(if_num, wl_rank, rankx); + } +#if WLEXTRAS_PATCH + // if there are unused entries to be filled + if ((rank_mask & 0x0F) != 0x0F) { + if (rankx < 3) { + debug("N%d.LMC%d.R%d: checking for WLEVEL_RANK unused entries.\n", + node, if_num, rankx); + + // if rank 0, write ranks 1 and 2 here if empty + if (rankx == 0) { + // check that rank 1 is empty + if (!(rank_mask & (1 << 1))) { + debug("N%d.LMC%d.R%d: writing WLEVEL_RANK unused entry R%d.\n", + node, if_num, rankx, 1); + lmc_wr(priv, + CVMX_LMCX_WLEVEL_RANKX(1, + if_num), + wl_rank.u64); + } + + // check that rank 2 is empty + if (!(rank_mask & (1 << 2))) { + debug("N%d.LMC%d.R%d: writing WLEVEL_RANK unused entry R%d.\n", + node, if_num, rankx, 2); + lmc_wr(priv, + CVMX_LMCX_WLEVEL_RANKX(2, + if_num), + wl_rank.u64); + } + } + + // if rank 0, 1 or 2, write rank 3 here if empty + // check that rank 3 is empty + if (!(rank_mask & (1 << 3))) { + debug("N%d.LMC%d.R%d: writing WLEVEL_RANK unused entry R%d.\n", + node, if_num, rankx, 3); + lmc_wr(priv, + CVMX_LMCX_WLEVEL_RANKX(3, + if_num), + wl_rank.u64); + } + } + } +#endif /* WLEXTRAS_PATCH */ + } + + /* Enable 32-bit mode if required. */ + cfg.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + cfg.cn78xx.mode32b = (!if_64b); + debug("%-45s : %d\n", "MODE32B", cfg.cn78xx.mode32b); + + /* Restore the ECC configuration */ + if (!sw_wl_hw_default) + cfg.cn78xx.ecc_ena = use_ecc; + + lmc_wr(priv, CVMX_LMCX_CONFIG(if_num), cfg.u64); + + return 0; +} + +static void lmc_dll(struct ddr_priv *priv) +{ +#ifdef CAVIUM_ONLY + union cvmx_lmcx_dll_ctl3 ddr_dll_ctl3; + int setting[9]; + int i; + + ddr_dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); + + for (i = 0; i < 9; ++i) { + SET_DDR_DLL_CTL3(dll90_byte_sel, ENCODE_DLL90_BYTE_SEL(i)); + lmc_wr(priv, CVMX_LMCX_DLL_CTL3(if_num), ddr_dll_ctl3.u64); + lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); + ddr_dll_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL3(if_num)); + setting[i] = GET_DDR_DLL_CTL3(dll90_setting); + debug("%d. LMC%d_DLL_CTL3[%d] = %016llx %d\n", i, if_num, + GET_DDR_DLL_CTL3(dll90_byte_sel), ddr_dll_ctl3.u64, + setting[i]); + } + + debug("N%d.LMC%d: %-36s : %5d %5d %5d %5d %5d %5d %5d %5d %5d\n", + node, if_num, "DLL90 Setting 8:0", + setting[8], setting[7], setting[6], setting[5], setting[4], + setting[3], setting[2], setting[1], setting[0]); +#endif /* CAVIUM_ONLY */ + + process_custom_dll_offsets(priv, if_num, "ddr_dll_write_offset", + c_cfg->dll_write_offset, + "ddr%d_dll_write_offset_byte%d", 1); + process_custom_dll_offsets(priv, if_num, "ddr_dll_read_offset", + c_cfg->dll_read_offset, + "ddr%d_dll_read_offset_byte%d", 2); +} + +#define SLOT_CTL_INCR(csr, chip, field, incr) \ + csr.chip.field = (csr.chip.field < (64 - incr)) ? \ + (csr.chip.field + incr) : 63 + +#define INCR(csr, chip, field, incr) \ + csr.chip.field = (csr.chip.field < (64 - incr)) ? \ + (csr.chip.field + incr) : 63 + +static void lmc_workaround_2(struct ddr_priv *priv) +{ + /* Workaround Errata 21063 */ + if (octeon_is_cpuid(OCTEON_CN78XX) || + octeon_is_cpuid(OCTEON_CN70XX_PASS1_X)) { + union cvmx_lmcx_slot_ctl0 slot_ctl0; + union cvmx_lmcx_slot_ctl1 slot_ctl1; + union cvmx_lmcx_slot_ctl2 slot_ctl2; + union cvmx_lmcx_ext_config ext_cfg; + + slot_ctl0.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL0(if_num)); + slot_ctl1.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL1(if_num)); + slot_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL2(if_num)); + + ext_cfg.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(if_num)); + + /* When ext_cfg.s.read_ena_bprch is set add 1 */ + if (ext_cfg.s.read_ena_bprch) { + SLOT_CTL_INCR(slot_ctl0, cn78xx, r2w_init, 1); + SLOT_CTL_INCR(slot_ctl0, cn78xx, r2w_l_init, 1); + SLOT_CTL_INCR(slot_ctl1, cn78xx, r2w_xrank_init, 1); + SLOT_CTL_INCR(slot_ctl2, cn78xx, r2w_xdimm_init, 1); + } + + /* Always add 2 */ + SLOT_CTL_INCR(slot_ctl1, cn78xx, w2r_xrank_init, 2); + SLOT_CTL_INCR(slot_ctl2, cn78xx, w2r_xdimm_init, 2); + + lmc_wr(priv, CVMX_LMCX_SLOT_CTL0(if_num), slot_ctl0.u64); + lmc_wr(priv, CVMX_LMCX_SLOT_CTL1(if_num), slot_ctl1.u64); + lmc_wr(priv, CVMX_LMCX_SLOT_CTL2(if_num), slot_ctl2.u64); + } + + /* Workaround Errata 21216 */ + if (octeon_is_cpuid(OCTEON_CN78XX_PASS1_X) || + octeon_is_cpuid(OCTEON_CN70XX_PASS1_X)) { + union cvmx_lmcx_slot_ctl1 slot_ctl1; + union cvmx_lmcx_slot_ctl2 slot_ctl2; + + slot_ctl1.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL1(if_num)); + slot_ctl1.cn78xx.w2w_xrank_init = + max(10, (int)slot_ctl1.cn78xx.w2w_xrank_init); + lmc_wr(priv, CVMX_LMCX_SLOT_CTL1(if_num), slot_ctl1.u64); + + slot_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL2(if_num)); + slot_ctl2.cn78xx.w2w_xdimm_init = + max(10, (int)slot_ctl2.cn78xx.w2w_xdimm_init); + lmc_wr(priv, CVMX_LMCX_SLOT_CTL2(if_num), slot_ctl2.u64); + } +} + +static void lmc_final(struct ddr_priv *priv) +{ + /* + * 4.8.11 Final LMC Initialization + * + * Early LMC initialization, LMC write-leveling, and LMC read-leveling + * must be completed prior to starting this final LMC initialization. + * + * LMC hardware updates the LMC(0)_SLOT_CTL0, LMC(0)_SLOT_CTL1, + * LMC(0)_SLOT_CTL2 CSRs with minimum values based on the selected + * readleveling and write-leveling settings. Software should not write + * the final LMC(0)_SLOT_CTL0, LMC(0)_SLOT_CTL1, and LMC(0)_SLOT_CTL2 + * values until after the final read-leveling and write-leveling + * settings are written. + * + * Software must ensure the LMC(0)_SLOT_CTL0, LMC(0)_SLOT_CTL1, and + * LMC(0)_SLOT_CTL2 CSR values are appropriate for this step. These CSRs + * select the minimum gaps between read operations and write operations + * of various types. + * + * Software must not reduce the values in these CSR fields below the + * values previously selected by the LMC hardware (during write-leveling + * and read-leveling steps above). + * + * All sections in this chapter may be used to derive proper settings + * for these registers. + * + * For minimal read latency, L2C_CTL[EF_ENA,EF_CNT] should be programmed + * properly. This should be done prior to the first read. + */ + +#ifdef ENABLE_SLOT_CTL_ACCESS + union cvmx_lmcx_slot_ctl0 lmc_slot_ctl0; + union cvmx_lmcx_slot_ctl1 lmc_slot_ctl1; + union cvmx_lmcx_slot_ctl2 lmc_slot_ctl2; + union cvmx_lmcx_slot_ctl3 lmc_slot_ctl3; + + lmc_slot_ctl0.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL0(if_num)); + lmc_slot_ctl1.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL1(if_num)); + lmc_slot_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL2(if_num)); + lmc_slot_ctl3.u64 = lmc_rd(priv, CVMX_LMCX_SLOT_CTL3(if_num)); + + debug("%-45s : 0x%016llx\n", "LMC_SLOT_CTL0", lmc_slot_ctl0.u64); + debug("%-45s : 0x%016llx\n", "LMC_SLOT_CTL1", lmc_slot_ctl1.u64); + debug("%-45s : 0x%016llx\n", "LMC_SLOT_CTL2", lmc_slot_ctl2.u64); + debug("%-45s : 0x%016llx\n", "LMC_SLOT_CTL3", lmc_slot_ctl3.u64); + + // for now, look only for SLOT_CTL1 envvar for override of contents + s = lookup_env(priv, "ddr%d_slot_ctl1", if_num); + if (s) { + int slot_ctl1_incr = strtoul(s, NULL, 0); + // validate the value + if (slot_ctl1_incr < 1 || slot_ctl1_incr > 3) { + printf("ddr%d_slot_ctl1 illegal value (%d); must be 1-3\n", + if_num, slot_ctl1_incr); + } else { + // modify all the SLOT_CTL1 fields by the increment, + // for now... + // but make sure the value will not overflow!!! + INCR(lmc_slot_ctl1, s, r2r_xrank_init, slot_ctl1_incr); + INCR(lmc_slot_ctl1, s, r2w_xrank_init, slot_ctl1_incr); + INCR(lmc_slot_ctl1, s, w2r_xrank_init, slot_ctl1_incr); + INCR(lmc_slot_ctl1, s, w2w_xrank_init, slot_ctl1_incr); + lmc_wr(priv, CVMX_LMCX_SLOT_CTL1(if_num), + lmc_slot_ctl1.u64); + } + } +#endif /* ENABLE_SLOT_CTL_ACCESS */ + + /* Clear any residual ECC errors */ + int num_tads = 1; + int tad; + int num_mcis = 1; + int mci; + + if (octeon_is_cpuid(OCTEON_CN78XX)) { + num_tads = 8; + num_mcis = 4; + } else if (octeon_is_cpuid(OCTEON_CN70XX)) { + num_tads = 1; + num_mcis = 1; + } else if (octeon_is_cpuid(OCTEON_CN73XX) || + octeon_is_cpuid(OCTEON_CNF75XX)) { + num_tads = 4; + num_mcis = 3; + } + + lmc_wr(priv, CVMX_LMCX_INT(if_num), -1ULL); + lmc_rd(priv, CVMX_LMCX_INT(if_num)); + + for (tad = 0; tad < num_tads; tad++) { + l2c_wr(priv, CVMX_L2C_TADX_INT(tad), + l2c_rd(priv, CVMX_L2C_TADX_INT(tad))); + debug("%-45s : (%d) 0x%08llx\n", "CVMX_L2C_TAD_INT", tad, + l2c_rd(priv, CVMX_L2C_TADX_INT(tad))); + } + + for (mci = 0; mci < num_mcis; mci++) { + l2c_wr(priv, CVMX_L2C_MCIX_INT(mci), + l2c_rd(priv, CVMX_L2C_MCIX_INT(mci))); + debug("%-45s : (%d) 0x%08llx\n", "L2C_MCI_INT", mci, + l2c_rd(priv, CVMX_L2C_MCIX_INT(mci))); + } + + debug("%-45s : 0x%08llx\n", "LMC_INT", + lmc_rd(priv, CVMX_LMCX_INT(if_num))); +} + +static void lmc_scrambling(struct ddr_priv *priv) +{ + // Make sure scrambling is disabled during init... + union cvmx_lmcx_control ctrl; + union cvmx_lmcx_scramble_cfg0 lmc_scramble_cfg0; + union cvmx_lmcx_scramble_cfg1 lmc_scramble_cfg1; + union cvmx_lmcx_scramble_cfg2 lmc_scramble_cfg2; + union cvmx_lmcx_ns_ctl lmc_ns_ctl; + int use_scramble = 0; // default OFF + char *s; + + ctrl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + lmc_scramble_cfg0.u64 = lmc_rd(priv, CVMX_LMCX_SCRAMBLE_CFG0(if_num)); + lmc_scramble_cfg1.u64 = lmc_rd(priv, CVMX_LMCX_SCRAMBLE_CFG1(if_num)); + lmc_scramble_cfg2.u64 = 0; // quiet compiler + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) { + lmc_scramble_cfg2.u64 = + lmc_rd(priv, CVMX_LMCX_SCRAMBLE_CFG2(if_num)); + } + lmc_ns_ctl.u64 = lmc_rd(priv, CVMX_LMCX_NS_CTL(if_num)); + + s = lookup_env_ull(priv, "ddr_use_scramble"); + if (s) + use_scramble = simple_strtoull(s, NULL, 0); + + /* Generate random values if scrambling is needed */ + if (use_scramble) { + lmc_scramble_cfg0.u64 = cvmx_rng_get_random64(); + lmc_scramble_cfg1.u64 = cvmx_rng_get_random64(); + lmc_scramble_cfg2.u64 = cvmx_rng_get_random64(); + lmc_ns_ctl.s.ns_scramble_dis = 0; + lmc_ns_ctl.s.adr_offset = 0; + ctrl.s.scramble_ena = 1; + } + + s = lookup_env_ull(priv, "ddr_scramble_cfg0"); + if (s) { + lmc_scramble_cfg0.u64 = simple_strtoull(s, NULL, 0); + ctrl.s.scramble_ena = 1; + } + debug("%-45s : 0x%016llx\n", "LMC_SCRAMBLE_CFG0", + lmc_scramble_cfg0.u64); + + lmc_wr(priv, CVMX_LMCX_SCRAMBLE_CFG0(if_num), lmc_scramble_cfg0.u64); + + s = lookup_env_ull(priv, "ddr_scramble_cfg1"); + if (s) { + lmc_scramble_cfg1.u64 = simple_strtoull(s, NULL, 0); + ctrl.s.scramble_ena = 1; + } + debug("%-45s : 0x%016llx\n", "LMC_SCRAMBLE_CFG1", + lmc_scramble_cfg1.u64); + lmc_wr(priv, CVMX_LMCX_SCRAMBLE_CFG1(if_num), lmc_scramble_cfg1.u64); + + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) { + s = lookup_env_ull(priv, "ddr_scramble_cfg2"); + if (s) { + lmc_scramble_cfg2.u64 = simple_strtoull(s, NULL, 0); + ctrl.s.scramble_ena = 1; + } + debug("%-45s : 0x%016llx\n", "LMC_SCRAMBLE_CFG2", + lmc_scramble_cfg1.u64); + lmc_wr(priv, CVMX_LMCX_SCRAMBLE_CFG2(if_num), + lmc_scramble_cfg2.u64); + } + + s = lookup_env_ull(priv, "ddr_ns_ctl"); + if (s) + lmc_ns_ctl.u64 = simple_strtoull(s, NULL, 0); + debug("%-45s : 0x%016llx\n", "LMC_NS_CTL", lmc_ns_ctl.u64); + lmc_wr(priv, CVMX_LMCX_NS_CTL(if_num), lmc_ns_ctl.u64); + + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctrl.u64); +} + +struct rl_score { + u64 setting; + int score; +}; + +static union cvmx_lmcx_rlevel_rankx rl_rank __section(".data"); +static union cvmx_lmcx_rlevel_ctl rl_ctl __section(".data"); +static unsigned char rodt_ctl __section(".data"); + +static int rl_rodt_err __section(".data"); +static unsigned char rtt_nom __section(".data"); +static unsigned char rtt_idx __section(".data"); +static char min_rtt_nom_idx __section(".data"); +static char max_rtt_nom_idx __section(".data"); +static char min_rodt_ctl __section(".data"); +static char max_rodt_ctl __section(".data"); +static int rl_dbg_loops __section(".data"); +static unsigned char save_ddr2t __section(".data"); +static int rl_samples __section(".data"); +static char rl_compute __section(".data"); +static char saved_ddr__ptune __section(".data"); +static char saved_ddr__ntune __section(".data"); +static char rl_comp_offs __section(".data"); +static char saved_int_zqcs_dis __section(".data"); +static int max_adj_rl_del_inc __section(".data"); +static int print_nom_ohms __section(".data"); +static int rl_print __section(".data"); + +#ifdef ENABLE_HARDCODED_RLEVEL +static char part_number[21] __section(".data"); +#endif /* ENABLE_HARDCODED_RLEVEL */ + +#if PERFECT_BITMASK_COUNTING +struct perfect_counts { + u16 count[9][32]; // 8+ECC by 64 values + u32 mask[9]; // 8+ECC, bitmask of perfect delays +}; + +static struct perfect_counts rank_perf[4] __section(".data"); +static struct perfect_counts rodt_perfect_counts __section(".data"); +static int pbm_lowsum_limit __section(".data"); +// FIXME: PBM skip for RODT 240 and 34 +static u32 pbm_rodt_skip __section(".data"); +#endif /* PERFECT_BITMASK_COUNTING */ + +// control rank majority processing +static int disable_rank_majority __section(".data"); + +// default to mask 11b ODDs for DDR4 (except 73xx), else DISABLE +// for DDR3 +static int enable_rldelay_bump __section(".data"); +static int rldelay_bump_incr __section(".data"); +static int disable_rlv_bump_this_byte __section(".data"); +static u64 value_mask __section(".data"); + +static struct rlevel_byte_data rl_byte[9] __section(".data"); +static int sample_loops __section(".data"); +static int max_samples __section(".data"); +static int rl_rank_errors __section(".data"); +static int rl_mask_err __section(".data"); +static int rl_nonseq_err __section(".data"); +static struct rlevel_bitmask rl_mask[9] __section(".data"); +static int rl_best_rank_score __section(".data"); + +static int rodt_row_skip_mask __section(".data"); + +static void rodt_loop(struct ddr_priv *priv, int rankx, struct rl_score + rl_score[RTT_NOM_OHMS_COUNT][RODT_OHMS_COUNT][4]) +{ + union cvmx_lmcx_comp_ctl2 cc2; + const int rl_separate_ab = 1; + int i; + + rl_best_rank_score = DEFAULT_BEST_RANK_SCORE; + rl_rodt_err = 0; + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + cc2.cn78xx.rodt_ctl = rodt_ctl; + lmc_wr(priv, CVMX_LMCX_COMP_CTL2(if_num), cc2.u64); + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + udelay(1); /* Give it a little time to take affect */ + if (rl_print > 1) { + debug("Read ODT_CTL : 0x%x (%d ohms)\n", + cc2.cn78xx.rodt_ctl, + imp_val->rodt_ohms[cc2.cn78xx.rodt_ctl]); + } + + memset(rl_byte, 0, sizeof(rl_byte)); +#if PERFECT_BITMASK_COUNTING + memset(&rodt_perfect_counts, 0, sizeof(rodt_perfect_counts)); +#endif /* PERFECT_BITMASK_COUNTING */ + + // when iter RODT is the target RODT, take more samples... + max_samples = rl_samples; + if (rodt_ctl == default_rodt_ctl) + max_samples += rl_samples + 1; + + for (sample_loops = 0; sample_loops < max_samples; sample_loops++) { + int redoing_nonseq_errs = 0; + + rl_mask_err = 0; + + if (!(rl_separate_ab && spd_rdimm && + ddr_type == DDR4_DRAM)) { + /* Clear read-level delays */ + lmc_wr(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num), 0); + + /* read-leveling */ + oct3_ddr3_seq(priv, 1 << rankx, if_num, 1); + + do { + rl_rank.u64 = + lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + } while (rl_rank.cn78xx.status != 3); + } + + rl_rank.u64 = + lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num)); + + // start bitmask interpretation block + + memset(rl_mask, 0, sizeof(rl_mask)); + + if (rl_separate_ab && spd_rdimm && ddr_type == DDR4_DRAM) { + union cvmx_lmcx_rlevel_rankx rl_rank_aside; + union cvmx_lmcx_modereg_params0 mp0; + + /* A-side */ + mp0.u64 = + lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num)); + mp0.s.mprloc = 0; /* MPR Page 0 Location 0 */ + lmc_wr(priv, + CVMX_LMCX_MODEREG_PARAMS0(if_num), + mp0.u64); + + /* Clear read-level delays */ + lmc_wr(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num), 0); + + /* read-leveling */ + oct3_ddr3_seq(priv, 1 << rankx, if_num, 1); + + do { + rl_rank.u64 = + lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + } while (rl_rank.cn78xx.status != 3); + + rl_rank.u64 = + lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + + rl_rank_aside.u64 = rl_rank.u64; + + rl_mask[0].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 0); + rl_mask[1].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 1); + rl_mask[2].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 2); + rl_mask[3].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 3); + rl_mask[8].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 8); + /* A-side complete */ + + /* B-side */ + mp0.u64 = + lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num)); + mp0.s.mprloc = 3; /* MPR Page 0 Location 3 */ + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num), + mp0.u64); + + /* Clear read-level delays */ + lmc_wr(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num), 0); + + /* read-leveling */ + oct3_ddr3_seq(priv, 1 << rankx, if_num, 1); + + do { + rl_rank.u64 = + lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + } while (rl_rank.cn78xx.status != 3); + + rl_rank.u64 = + lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + + rl_mask[4].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 4); + rl_mask[5].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 5); + rl_mask[6].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 6); + rl_mask[7].bm = lmc_ddr3_rl_dbg_read(priv, if_num, 7); + /* B-side complete */ + + upd_rl_rank(&rl_rank, 0, rl_rank_aside.s.byte0); + upd_rl_rank(&rl_rank, 1, rl_rank_aside.s.byte1); + upd_rl_rank(&rl_rank, 2, rl_rank_aside.s.byte2); + upd_rl_rank(&rl_rank, 3, rl_rank_aside.s.byte3); + /* ECC A-side */ + upd_rl_rank(&rl_rank, 8, rl_rank_aside.s.byte8); + + mp0.u64 = + lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num)); + mp0.s.mprloc = 0; /* MPR Page 0 Location 0 */ + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(if_num), + mp0.u64); + } + + /* + * Evaluate the quality of the read-leveling delays from the + * bitmasks. Also save off a software computed read-leveling + * mask that may be used later to qualify the delay results + * from Octeon. + */ + for (i = 0; i < (8 + ecc_ena); ++i) { + int bmerr; + + if (!(if_bytemask & (1 << i))) + continue; + if (!(rl_separate_ab && spd_rdimm && + ddr_type == DDR4_DRAM)) { + rl_mask[i].bm = + lmc_ddr3_rl_dbg_read(priv, if_num, i); + } + bmerr = validate_ddr3_rlevel_bitmask(&rl_mask[i], + ddr_type); + rl_mask[i].errs = bmerr; + rl_mask_err += bmerr; +#if PERFECT_BITMASK_COUNTING + // count only the "perfect" bitmasks + if (ddr_type == DDR4_DRAM && !bmerr) { + int delay; + // FIXME: for now, simple filtering: + // do NOT count PBMs for RODTs in skip mask + if ((1U << rodt_ctl) & pbm_rodt_skip) + continue; + // FIXME: could optimize this a bit? + delay = get_rl_rank(&rl_rank, i); + rank_perf[rankx].count[i][delay] += 1; + rank_perf[rankx].mask[i] |= + (1ULL << delay); + rodt_perfect_counts.count[i][delay] += 1; + rodt_perfect_counts.mask[i] |= (1ULL << delay); + } +#endif /* PERFECT_BITMASK_COUNTING */ + } + + /* Set delays for unused bytes to match byte 0. */ + for (i = 0; i < 9; ++i) { + if (if_bytemask & (1 << i)) + continue; + upd_rl_rank(&rl_rank, i, rl_rank.s.byte0); + } + + /* + * Save a copy of the byte delays in physical + * order for sequential evaluation. + */ + unpack_rlevel_settings(if_bytemask, ecc_ena, rl_byte, rl_rank); + + redo_nonseq_errs: + + rl_nonseq_err = 0; + if (!disable_sequential_delay_check) { + for (i = 0; i < 9; ++i) + rl_byte[i].sqerrs = 0; + + if ((if_bytemask & 0xff) == 0xff) { + /* + * Evaluate delay sequence across the whole + * range of bytes for standard dimms. + */ + /* 1=RDIMM, 5=Mini-RDIMM */ + if (spd_dimm_type == 1 || spd_dimm_type == 5) { + int reg_adj_del = abs(rl_byte[4].delay - + rl_byte[5].delay); + + /* + * Registered dimm topology routes + * from the center. + */ + rl_nonseq_err += + nonseq_del(rl_byte, 0, + 3 + ecc_ena, + max_adj_rl_del_inc); + rl_nonseq_err += + nonseq_del(rl_byte, 5, + 7 + ecc_ena, + max_adj_rl_del_inc); + // byte 5 sqerrs never gets cleared + // for RDIMMs + rl_byte[5].sqerrs = 0; + if (reg_adj_del > 1) { + /* + * Assess proximity of bytes on + * opposite sides of register + */ + rl_nonseq_err += (reg_adj_del - + 1) * + RLEVEL_ADJACENT_DELAY_ERROR; + // update byte 5 error + rl_byte[5].sqerrs += + (reg_adj_del - 1) * + RLEVEL_ADJACENT_DELAY_ERROR; + } + } + + /* 2=UDIMM, 6=Mini-UDIMM */ + if (spd_dimm_type == 2 || spd_dimm_type == 6) { + /* + * Unbuffered dimm topology routes + * from end to end. + */ + rl_nonseq_err += nonseq_del(rl_byte, 0, + 7 + ecc_ena, + max_adj_rl_del_inc); + } + } else { + rl_nonseq_err += nonseq_del(rl_byte, 0, + 3 + ecc_ena, + max_adj_rl_del_inc); + } + } /* if (! disable_sequential_delay_check) */ + + rl_rank_errors = rl_mask_err + rl_nonseq_err; + + // print original sample here only if we are not really + // averaging or picking best + // also do not print if we were redoing the NONSEQ score + // for using COMPUTED + if (!redoing_nonseq_errs && rl_samples < 2) { + if (rl_print > 1) { + display_rl_bm(if_num, rankx, rl_mask, ecc_ena); + display_rl_bm_scores(if_num, rankx, rl_mask, + ecc_ena); + display_rl_seq_scores(if_num, rankx, rl_byte, + ecc_ena); + } + display_rl_with_score(if_num, rl_rank, rankx, + rl_rank_errors); + } + + if (rl_compute) { + if (!redoing_nonseq_errs) { + /* Recompute the delays based on the bitmask */ + for (i = 0; i < (8 + ecc_ena); ++i) { + if (!(if_bytemask & (1 << i))) + continue; + + upd_rl_rank(&rl_rank, i, + compute_ddr3_rlevel_delay( + rl_mask[i].mstart, + rl_mask[i].width, + rl_ctl)); + } + + /* + * Override the copy of byte delays with the + * computed results. + */ + unpack_rlevel_settings(if_bytemask, ecc_ena, + rl_byte, rl_rank); + + redoing_nonseq_errs = 1; + goto redo_nonseq_errs; + + } else { + /* + * now print this if already printed the + * original sample + */ + if (rl_samples < 2 || rl_print) { + display_rl_with_computed(if_num, + rl_rank, rankx, + rl_rank_errors); + } + } + } /* if (rl_compute) */ + + // end bitmask interpretation block + + // if it is a better (lower) score, then keep it + if (rl_rank_errors < rl_best_rank_score) { + rl_best_rank_score = rl_rank_errors; + + // save the new best delays and best errors + for (i = 0; i < (8 + ecc_ena); ++i) { + rl_byte[i].best = rl_byte[i].delay; + rl_byte[i].bestsq = rl_byte[i].sqerrs; + // save bitmasks and their scores as well + // xlate UNPACKED index to PACKED index to + // get from rl_mask + rl_byte[i].bm = rl_mask[XUP(i, !!ecc_ena)].bm; + rl_byte[i].bmerrs = + rl_mask[XUP(i, !!ecc_ena)].errs; + } + } + + rl_rodt_err += rl_rank_errors; + } + + /* We recorded the best score across the averaging loops */ + rl_score[rtt_nom][rodt_ctl][rankx].score = rl_best_rank_score; + + /* + * Restore the delays from the best fields that go with the best + * score + */ + for (i = 0; i < 9; ++i) { + rl_byte[i].delay = rl_byte[i].best; + rl_byte[i].sqerrs = rl_byte[i].bestsq; + } + + rl_rank.u64 = lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num)); + + pack_rlevel_settings(if_bytemask, ecc_ena, rl_byte, &rl_rank); + + if (rl_samples > 1) { + // restore the "best" bitmasks and their scores for printing + for (i = 0; i < 9; ++i) { + if ((if_bytemask & (1 << i)) == 0) + continue; + // xlate PACKED index to UNPACKED index to get from + // rl_byte + rl_mask[i].bm = rl_byte[XPU(i, !!ecc_ena)].bm; + rl_mask[i].errs = rl_byte[XPU(i, !!ecc_ena)].bmerrs; + } + + // maybe print bitmasks/scores here + if (rl_print > 1) { + display_rl_bm(if_num, rankx, rl_mask, ecc_ena); + display_rl_bm_scores(if_num, rankx, rl_mask, ecc_ena); + display_rl_seq_scores(if_num, rankx, rl_byte, ecc_ena); + + display_rl_with_rodt(if_num, rl_rank, rankx, + rl_score[rtt_nom][rodt_ctl][rankx].score, + print_nom_ohms, + imp_val->rodt_ohms[rodt_ctl], + WITH_RODT_BESTSCORE); + + debug("-----------\n"); + } + } + + rl_score[rtt_nom][rodt_ctl][rankx].setting = rl_rank.u64; + +#if PERFECT_BITMASK_COUNTING + // print out the PBMs for the current RODT + if (ddr_type == DDR4_DRAM && rl_print > 1) { // verbosity? +#if PRINT_PERFECT_COUNTS + // FIXME: change verbosity level after debug complete... + + for (i = 0; i < 9; i++) { + u64 temp_mask; + int num_values; + + // FIXME: PBM skip for RODTs in mask + if ((1U << rodt_ctl) & pbm_rodt_skip) + continue; + + temp_mask = rodt_perfect_counts.mask[i]; + num_values = __builtin_popcountll(temp_mask); + i = __builtin_ffsll(temp_mask) - 1; + + debug("N%d.LMC%d.R%d: PERFECT: RODT %3d: Byte %d: mask 0x%02llx (%d): ", + node, if_num, rankx, + imp_val->rodt_ohms[rodt_ctl], + i, temp_mask >> i, num_values); + + while (temp_mask != 0) { + i = __builtin_ffsll(temp_mask) - 1; + debug("%2d(%2d) ", i, + rodt_perfect_counts.count[i][i]); + temp_mask &= ~(1UL << i); + } /* while (temp_mask != 0) */ + debug("\n"); + } +#endif + } +#endif /* PERFECT_BITMASK_COUNTING */ +} + +static void rank_major_loop(struct ddr_priv *priv, int rankx, struct rl_score + rl_score[RTT_NOM_OHMS_COUNT][RODT_OHMS_COUNT][4]) +{ + /* Start with an arbitrarily high score */ + int best_rank_score = DEFAULT_BEST_RANK_SCORE; + int best_rank_rtt_nom = 0; + int best_rank_ctl = 0; + int best_rank_ohms = 0; + int best_rankx = 0; + int dimm_rank_mask; + int max_rank_score; + union cvmx_lmcx_rlevel_rankx saved_rl_rank; + int next_ohms; + int orankx; + int next_score = 0; + int best_byte, new_byte, temp_byte, orig_best_byte; + int rank_best_bytes[9]; + int byte_sh; + int avg_byte; + int avg_diff; + int i; + + if (!(rank_mask & (1 << rankx))) + return; + + // some of the rank-related loops below need to operate only on + // the ranks of a single DIMM, + // so create a mask for their use here + if (num_ranks == 4) { + dimm_rank_mask = rank_mask; // should be 1111 + } else { + dimm_rank_mask = rank_mask & 3; // should be 01 or 11 + if (rankx >= 2) { + // doing a rank on the second DIMM, should be + // 0100 or 1100 + dimm_rank_mask <<= 2; + } + } + debug("DIMM rank mask: 0x%x, rank mask: 0x%x, rankx: %d\n", + dimm_rank_mask, rank_mask, rankx); + + // this is the start of the BEST ROW SCORE LOOP + + for (rtt_idx = min_rtt_nom_idx; rtt_idx <= max_rtt_nom_idx; ++rtt_idx) { + rtt_nom = imp_val->rtt_nom_table[rtt_idx]; + + debug("N%d.LMC%d.R%d: starting RTT_NOM %d (%d)\n", + node, if_num, rankx, rtt_nom, + imp_val->rtt_nom_ohms[rtt_nom]); + + for (rodt_ctl = max_rodt_ctl; rodt_ctl >= min_rodt_ctl; + --rodt_ctl) { + next_ohms = imp_val->rodt_ohms[rodt_ctl]; + + // skip RODT rows in mask, but *NOT* rows with too + // high a score; + // we will not use the skipped ones for printing or + // evaluating, but we need to allow all the + // non-skipped ones to be candidates for "best" + if (((1 << rodt_ctl) & rodt_row_skip_mask) != 0) { + debug("N%d.LMC%d.R%d: SKIPPING rodt:%d (%d) with rank_score:%d\n", + node, if_num, rankx, rodt_ctl, + next_ohms, next_score); + continue; + } + + // this is ROFFIX-0528 + for (orankx = 0; orankx < dimm_count * 4; orankx++) { + // stay on the same DIMM + if (!(dimm_rank_mask & (1 << orankx))) + continue; + + next_score = rl_score[rtt_nom][rodt_ctl][orankx].score; + + // always skip a higher score + if (next_score > best_rank_score) + continue; + + // if scores are equal + if (next_score == best_rank_score) { + // always skip lower ohms + if (next_ohms < best_rank_ohms) + continue; + + // if same ohms + if (next_ohms == best_rank_ohms) { + // always skip the other rank(s) + if (orankx != rankx) + continue; + } + // else next_ohms are greater, + // always choose it + } + // else next_score is less than current best, + // so always choose it + debug("N%d.LMC%d.R%d: new best score: rank %d, rodt %d(%3d), new best %d, previous best %d(%d)\n", + node, if_num, rankx, orankx, rodt_ctl, next_ohms, next_score, + best_rank_score, best_rank_ohms); + best_rank_score = next_score; + best_rank_rtt_nom = rtt_nom; + //best_rank_nom_ohms = rtt_nom_ohms; + best_rank_ctl = rodt_ctl; + best_rank_ohms = next_ohms; + best_rankx = orankx; + rl_rank.u64 = + rl_score[rtt_nom][rodt_ctl][orankx].setting; + } + } + } + + // this is the end of the BEST ROW SCORE LOOP + + // DANGER, Will Robinson!! Abort now if we did not find a best + // score at all... + if (best_rank_score == DEFAULT_BEST_RANK_SCORE) { + printf("N%d.LMC%d.R%d: WARNING: no best rank score found - resetting node...\n", + node, if_num, rankx); + mdelay(500); + do_reset(NULL, 0, 0, NULL); + } + + // FIXME: relative now, but still arbitrary... + max_rank_score = best_rank_score; + if (ddr_type == DDR4_DRAM) { + // halve the range if 2 DIMMs unless they are single rank... + max_rank_score += (MAX_RANK_SCORE_LIMIT / ((num_ranks > 1) ? + dimm_count : 1)); + } else { + // Since DDR3 typically has a wider score range, + // keep more of them always + max_rank_score += MAX_RANK_SCORE_LIMIT; + } + + if (!ecc_ena) { + /* ECC is not used */ + rl_rank.s.byte8 = rl_rank.s.byte0; + } + + // at the end, write the best row settings to the current rank + lmc_wr(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num), rl_rank.u64); + rl_rank.u64 = lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num)); + + saved_rl_rank.u64 = rl_rank.u64; + + // this is the start of the PRINT LOOP + int pass; + + // for pass==0, print current rank, pass==1 print other rank(s) + // this is done because we want to show each ranks RODT values + // together, not interlaced +#if COUNT_RL_CANDIDATES + // keep separates for ranks - pass=0 target rank, pass=1 other + // rank on DIMM + int mask_skipped[2] = {0, 0}; + int score_skipped[2] = {0, 0}; + int selected_rows[2] = {0, 0}; + int zero_scores[2] = {0, 0}; +#endif /* COUNT_RL_CANDIDATES */ + for (pass = 0; pass < 2; pass++) { + for (orankx = 0; orankx < dimm_count * 4; orankx++) { + // stay on the same DIMM + if (!(dimm_rank_mask & (1 << orankx))) + continue; + + if ((pass == 0 && orankx != rankx) || + (pass != 0 && orankx == rankx)) + continue; + + for (rtt_idx = min_rtt_nom_idx; + rtt_idx <= max_rtt_nom_idx; ++rtt_idx) { + rtt_nom = imp_val->rtt_nom_table[rtt_idx]; + if (dyn_rtt_nom_mask == 0) { + print_nom_ohms = -1; + } else { + print_nom_ohms = + imp_val->rtt_nom_ohms[rtt_nom]; + } + + // cycle through all the RODT values... + for (rodt_ctl = max_rodt_ctl; + rodt_ctl >= min_rodt_ctl; --rodt_ctl) { + union cvmx_lmcx_rlevel_rankx + temp_rl_rank; + int temp_score = + rl_score[rtt_nom][rodt_ctl][orankx].score; + int skip_row; + + temp_rl_rank.u64 = + rl_score[rtt_nom][rodt_ctl][orankx].setting; + + // skip RODT rows in mask, or rows + // with too high a score; + // we will not use them for printing + // or evaluating... +#if COUNT_RL_CANDIDATES + if ((1 << rodt_ctl) & + rodt_row_skip_mask) { + skip_row = WITH_RODT_SKIPPING; + ++mask_skipped[pass]; + } else if (temp_score > + max_rank_score) { + skip_row = WITH_RODT_SKIPPING; + ++score_skipped[pass]; + } else { + skip_row = WITH_RODT_BLANK; + ++selected_rows[pass]; + if (temp_score == 0) + ++zero_scores[pass]; + } + +#else /* COUNT_RL_CANDIDATES */ + skip_row = (((1 << rodt_ctl) & + rodt_row_skip_mask) || + (temp_score > + max_rank_score)) + ? WITH_RODT_SKIPPING : + WITH_RODT_BLANK; +#endif + // identify and print the BEST ROW + // when it comes up + if (skip_row == WITH_RODT_BLANK && + best_rankx == orankx && + best_rank_rtt_nom == rtt_nom && + best_rank_ctl == rodt_ctl) + skip_row = WITH_RODT_BESTROW; + + if (rl_print) { + display_rl_with_rodt(if_num, + temp_rl_rank, orankx, temp_score, + print_nom_ohms, + imp_val->rodt_ohms[rodt_ctl], + skip_row); + } + } + } + } + } +#if COUNT_RL_CANDIDATES + debug("N%d.LMC%d.R%d: RLROWS: selected %d+%d, zero_scores %d+%d, mask_skipped %d+%d, score_skipped %d+%d\n", + node, if_num, rankx, selected_rows[0], selected_rows[1], + zero_scores[0], zero_scores[1], mask_skipped[0], mask_skipped[1], + score_skipped[0], score_skipped[1]); +#endif /* COUNT_RL_CANDIDATES */ + // this is the end of the PRINT LOOP + + // now evaluate which bytes need adjusting + // collect the new byte values; first init with current best for + // neighbor use + for (i = 0, byte_sh = 0; i < 8 + ecc_ena; i++, byte_sh += 6) { + rank_best_bytes[i] = (int)(rl_rank.u64 >> byte_sh) & + RLEVEL_BYTE_MSK; + } + + // this is the start of the BEST BYTE LOOP + + for (i = 0, byte_sh = 0; i < 8 + ecc_ena; i++, byte_sh += 6) { + int sum = 0, count = 0; + int count_less = 0, count_same = 0, count_more = 0; +#if FAILSAFE_CHECK + int count_byte; // save the value we counted around +#endif /* FAILSAFE_CHECK */ + // for rank majority use + int rank_less = 0, rank_same = 0, rank_more = 0; + int neighbor; + int neigh_byte; + + best_byte = rank_best_bytes[i]; + orig_best_byte = rank_best_bytes[i]; + + // this is the start of the BEST BYTE AVERAGING LOOP + + // validate the initial "best" byte by looking at the + // average of the unskipped byte-column entries + // we want to do this before we go further, so we can + // try to start with a better initial value + // this is the so-called "BESTBUY" patch set + + for (rtt_idx = min_rtt_nom_idx; rtt_idx <= max_rtt_nom_idx; + ++rtt_idx) { + rtt_nom = imp_val->rtt_nom_table[rtt_idx]; + + for (rodt_ctl = max_rodt_ctl; rodt_ctl >= min_rodt_ctl; + --rodt_ctl) { + union cvmx_lmcx_rlevel_rankx temp_rl_rank; + int temp_score; + + // average over all the ranks + for (orankx = 0; orankx < dimm_count * 4; + orankx++) { + // stay on the same DIMM + if (!(dimm_rank_mask & (1 << orankx))) + continue; + + temp_score = + rl_score[rtt_nom][rodt_ctl][orankx].score; + // skip RODT rows in mask, or rows with + // too high a score; + // we will not use them for printing or + // evaluating... + + if (!((1 << rodt_ctl) & + rodt_row_skip_mask) && + temp_score <= max_rank_score) { + temp_rl_rank.u64 = + rl_score[rtt_nom][rodt_ctl][orankx].setting; + temp_byte = + (int)(temp_rl_rank.u64 >> byte_sh) & + RLEVEL_BYTE_MSK; + sum += temp_byte; + count++; + } + } + } + } + + // this is the end of the BEST BYTE AVERAGING LOOP + + // FIXME: validate count and sum?? + avg_byte = (int)divide_nint(sum, count); + avg_diff = best_byte - avg_byte; + new_byte = best_byte; + if (avg_diff != 0) { + // bump best up/dn by 1, not necessarily all the + // way to avg + new_byte = best_byte + ((avg_diff > 0) ? -1 : 1); + } + + if (rl_print) { + debug("N%d.LMC%d.R%d: START: Byte %d: best %d is different by %d from average %d, using %d.\n", + node, if_num, rankx, + i, best_byte, avg_diff, avg_byte, new_byte); + } + best_byte = new_byte; +#if FAILSAFE_CHECK + count_byte = new_byte; // save the value we will count around +#endif /* FAILSAFE_CHECK */ + + // At this point best_byte is either: + // 1. the original byte-column value from the best scoring + // RODT row, OR + // 2. that value bumped toward the average of all the + // byte-column values + // + // best_byte will not change from here on... + + // this is the start of the BEST BYTE COUNTING LOOP + + // NOTE: we do this next loop separately from above, because + // we count relative to "best_byte" + // which may have been modified by the above averaging + // operation... + + for (rtt_idx = min_rtt_nom_idx; rtt_idx <= max_rtt_nom_idx; + ++rtt_idx) { + rtt_nom = imp_val->rtt_nom_table[rtt_idx]; + + for (rodt_ctl = max_rodt_ctl; rodt_ctl >= min_rodt_ctl; + --rodt_ctl) { + union cvmx_lmcx_rlevel_rankx temp_rl_rank; + int temp_score; + + for (orankx = 0; orankx < dimm_count * 4; + orankx++) { // count over all the ranks + // stay on the same DIMM + if (!(dimm_rank_mask & (1 << orankx))) + continue; + + temp_score = + rl_score[rtt_nom][rodt_ctl][orankx].score; + // skip RODT rows in mask, or rows + // with too high a score; + // we will not use them for printing + // or evaluating... + if (((1 << rodt_ctl) & + rodt_row_skip_mask) || + temp_score > max_rank_score) + continue; + + temp_rl_rank.u64 = + rl_score[rtt_nom][rodt_ctl][orankx].setting; + temp_byte = (temp_rl_rank.u64 >> + byte_sh) & RLEVEL_BYTE_MSK; + + if (temp_byte == 0) + ; // do not count it if illegal + else if (temp_byte == best_byte) + count_same++; + else if (temp_byte == best_byte - 1) + count_less++; + else if (temp_byte == best_byte + 1) + count_more++; + // else do not count anything more + // than 1 away from the best + + // no rank counting if disabled + if (disable_rank_majority) + continue; + + // FIXME? count is relative to + // best_byte; should it be rank-based? + // rank counts only on main rank + if (orankx != rankx) + continue; + else if (temp_byte == best_byte) + rank_same++; + else if (temp_byte == best_byte - 1) + rank_less++; + else if (temp_byte == best_byte + 1) + rank_more++; + } + } + } + + if (rl_print) { + debug("N%d.LMC%d.R%d: COUNT: Byte %d: orig %d now %d, more %d same %d less %d (%d/%d/%d)\n", + node, if_num, rankx, + i, orig_best_byte, best_byte, + count_more, count_same, count_less, + rank_more, rank_same, rank_less); + } + + // this is the end of the BEST BYTE COUNTING LOOP + + // choose the new byte value + // we need to check that there is no gap greater than 2 + // between adjacent bytes (adjacency depends on DIMM type) + // use the neighbor value to help decide + // initially, the rank_best_bytes[] will contain values from + // the chosen lowest score rank + new_byte = 0; + + // neighbor is index-1 unless we are index 0 or index 8 (ECC) + neighbor = (i == 8) ? 3 : ((i == 0) ? 1 : i - 1); + neigh_byte = rank_best_bytes[neighbor]; + + // can go up or down or stay the same, so look at a numeric + // average to help + new_byte = (int)divide_nint(((count_more * (best_byte + 1)) + + (count_same * (best_byte + 0)) + + (count_less * (best_byte - 1))), + max(1, (count_more + count_same + + count_less))); + + // use neighbor to help choose with average + if (i > 0 && (abs(neigh_byte - new_byte) > 2) && + !disable_sequential_delay_check) { + // but not for byte 0 + int avg_pick = new_byte; + + if ((new_byte - best_byte) != 0) { + // back to best, average did not get better + new_byte = best_byte; + } else { + // avg was the same, still too far, now move + // it towards the neighbor + new_byte += (neigh_byte > new_byte) ? 1 : -1; + } + + if (rl_print) { + debug("N%d.LMC%d.R%d: AVERAGE: Byte %d: neighbor %d too different %d from average %d, picking %d.\n", + node, if_num, rankx, + i, neighbor, neigh_byte, avg_pick, + new_byte); + } + } else { + // NOTE: + // For now, we let the neighbor processing above trump + // the new simple majority processing here. + // This is mostly because we have seen no smoking gun + // for a neighbor bad choice (yet?). + // Also note that we will ALWAYS be using byte 0 + // majority, because of the if clause above. + + // majority is dependent on the counts, which are + // relative to best_byte, so start there + int maj_byte = best_byte; + int rank_maj; + int rank_sum; + + if (count_more > count_same && + count_more > count_less) { + maj_byte++; + } else if (count_less > count_same && + count_less > count_more) { + maj_byte--; + } + + if (maj_byte != new_byte) { + // print only when majority choice is + // different from average + if (rl_print) { + debug("N%d.LMC%d.R%d: MAJORTY: Byte %d: picking majority of %d over average %d.\n", + node, if_num, rankx, i, maj_byte, + new_byte); + } + new_byte = maj_byte; + } else { + if (rl_print) { + debug("N%d.LMC%d.R%d: AVERAGE: Byte %d: picking average of %d.\n", + node, if_num, rankx, i, new_byte); + } + } + + if (!disable_rank_majority) { + // rank majority is dependent on the rank + // counts, which are relative to best_byte, + // so start there, and adjust according to the + // rank counts majority + rank_maj = best_byte; + if (rank_more > rank_same && + rank_more > rank_less) { + rank_maj++; + } else if (rank_less > rank_same && + rank_less > rank_more) { + rank_maj--; + } + rank_sum = rank_more + rank_same + rank_less; + + // now, let rank majority possibly rule over + // the current new_byte however we got it + if (rank_maj != new_byte) { // only if different + // Here is where we decide whether to + // completely apply RANK_MAJORITY or not + // ignore if less than + if (rank_maj < new_byte) { + if (rl_print) { + debug("N%d.LMC%d.R%d: RANKMAJ: Byte %d: LESS: NOT using %d over %d.\n", + node, if_num, + rankx, i, + rank_maj, + new_byte); + } + } else { + // For the moment, we do it + // ONLY when running 2-slot + // configs + // OR when rank_sum is big + // enough + if (dimm_count > 1 || + rank_sum > 2) { + // print only when rank + // majority choice is + // selected + if (rl_print) { + debug("N%d.LMC%d.R%d: RANKMAJ: Byte %d: picking %d over %d.\n", + node, + if_num, + rankx, + i, + rank_maj, + new_byte); + } + new_byte = rank_maj; + } else { + // FIXME: print some + // info when we could + // have chosen RANKMAJ + // but did not + if (rl_print) { + debug("N%d.LMC%d.R%d: RANKMAJ: Byte %d: NOT using %d over %d (best=%d,sum=%d).\n", + node, + if_num, + rankx, + i, + rank_maj, + new_byte, + best_byte, + rank_sum); + } + } + } + } + } /* if (!disable_rank_majority) */ + } +#if FAILSAFE_CHECK + // one last check: + // if new_byte is still count_byte, BUT there was no count + // for that value, DO SOMETHING!!! + // FIXME: go back to original best byte from the best row + if (new_byte == count_byte && count_same == 0) { + new_byte = orig_best_byte; + if (rl_print) { + debug("N%d.LMC%d.R%d: FAILSAF: Byte %d: going back to original %d.\n", + node, if_num, rankx, i, new_byte); + } + } +#endif /* FAILSAFE_CHECK */ +#if PERFECT_BITMASK_COUNTING + // Look at counts for "perfect" bitmasks (PBMs) if we had + // any for this byte-lane. + // Remember, we only counted for DDR4, so zero means none + // or DDR3, and we bypass this... + value_mask = rank_perf[rankx].mask[i]; + disable_rlv_bump_this_byte = 0; + + if (value_mask != 0 && rl_ctl.cn78xx.offset == 1) { + int i, delay_count, delay_max = 0, del_val = 0; + int num_values = __builtin_popcountll(value_mask); + int sum_counts = 0; + u64 temp_mask = value_mask; + + disable_rlv_bump_this_byte = 1; + i = __builtin_ffsll(temp_mask) - 1; + if (rl_print) + debug("N%d.LMC%d.R%d: PERFECT: Byte %d: OFF1: mask 0x%02llx (%d): ", + node, if_num, rankx, i, value_mask >> i, + num_values); + + while (temp_mask != 0) { + i = __builtin_ffsll(temp_mask) - 1; + delay_count = rank_perf[rankx].count[i][i]; + sum_counts += delay_count; + if (rl_print) + debug("%2d(%2d) ", i, delay_count); + if (delay_count >= delay_max) { + delay_max = delay_count; + del_val = i; + } + temp_mask &= ~(1UL << i); + } /* while (temp_mask != 0) */ + + // if sum_counts is small, just use NEW_BYTE + if (sum_counts < pbm_lowsum_limit) { + if (rl_print) + debug(": LOWSUM (%2d), choose ORIG ", + sum_counts); + del_val = new_byte; + delay_max = rank_perf[rankx].count[i][del_val]; + } + + // finish printing here... + if (rl_print) { + debug(": USING %2d (%2d) D%d\n", del_val, + delay_max, disable_rlv_bump_this_byte); + } + + new_byte = del_val; // override with best PBM choice + + } else if ((value_mask != 0) && (rl_ctl.cn78xx.offset == 2)) { + // if (value_mask != 0) { + int i, delay_count, del_val; + int num_values = __builtin_popcountll(value_mask); + int sum_counts = 0; + u64 temp_mask = value_mask; + + i = __builtin_ffsll(temp_mask) - 1; + if (rl_print) + debug("N%d.LMC%d.R%d: PERFECT: Byte %d: mask 0x%02llx (%d): ", + node, if_num, rankx, i, value_mask >> i, + num_values); + while (temp_mask != 0) { + i = __builtin_ffsll(temp_mask) - 1; + delay_count = rank_perf[rankx].count[i][i]; + sum_counts += delay_count; + if (rl_print) + debug("%2d(%2d) ", i, delay_count); + temp_mask &= ~(1UL << i); + } /* while (temp_mask != 0) */ + + del_val = __builtin_ffsll(value_mask) - 1; + delay_count = + rank_perf[rankx].count[i][del_val]; + + // overkill, normally only 1-4 bits + i = (value_mask >> del_val) & 0x1F; + + // if sum_counts is small, treat as special and use + // NEW_BYTE + if (sum_counts < pbm_lowsum_limit) { + if (rl_print) + debug(": LOWSUM (%2d), choose ORIG", + sum_counts); + i = 99; // SPECIAL case... + } + + switch (i) { + case 0x01 /* 00001b */: + // allow BUMP + break; + + case 0x13 /* 10011b */: + case 0x0B /* 01011b */: + case 0x03 /* 00011b */: + del_val += 1; // take the second + disable_rlv_bump_this_byte = 1; // allow no BUMP + break; + + case 0x0D /* 01101b */: + case 0x05 /* 00101b */: + // test count of lowest and all + if (delay_count >= 5 || sum_counts <= 5) + del_val += 1; // take the hole + else + del_val += 2; // take the next set + disable_rlv_bump_this_byte = 1; // allow no BUMP + break; + + case 0x0F /* 01111b */: + case 0x17 /* 10111b */: + case 0x07 /* 00111b */: + del_val += 1; // take the second + if (delay_count < 5) { // lowest count is small + int second = + rank_perf[rankx].count[i][del_val]; + int third = + rank_perf[rankx].count[i][del_val + 1]; + // test if middle is more than 1 OR + // top is more than 1; + // this means if they are BOTH 1, + // then we keep the second... + if (second > 1 || third > 1) { + // if middle is small OR top + // is large + if (second < 5 || + third > 1) { + // take the top + del_val += 1; + if (rl_print) + debug(": TOP7 "); + } + } + } + disable_rlv_bump_this_byte = 1; // allow no BUMP + break; + + default: // all others... + if (rl_print) + debug(": ABNORMAL, choose ORIG"); + + case 99: // special + // FIXME: choose original choice? + del_val = new_byte; + disable_rlv_bump_this_byte = 1; // allow no BUMP + break; + } + delay_count = + rank_perf[rankx].count[i][del_val]; + + // finish printing here... + if (rl_print) + debug(": USING %2d (%2d) D%d\n", del_val, + delay_count, disable_rlv_bump_this_byte); + new_byte = del_val; // override with best PBM choice + } else { + if (ddr_type == DDR4_DRAM) { // only report when DDR4 + // FIXME: remove or increase VBL for this + // output... + if (rl_print) + debug("N%d.LMC%d.R%d: PERFECT: Byte %d: ZERO PBMs, USING %d\n", + node, if_num, rankx, i, + new_byte); + // prevent ODD bump, rely on original + disable_rlv_bump_this_byte = 1; + } + } /* if (value_mask != 0) */ +#endif /* PERFECT_BITMASK_COUNTING */ + + // optionally bump the delay value + if (enable_rldelay_bump && !disable_rlv_bump_this_byte) { + if ((new_byte & enable_rldelay_bump) == + enable_rldelay_bump) { + int bump_value = new_byte + rldelay_bump_incr; + + if (rl_print) { + debug("N%d.LMC%d.R%d: RLVBUMP: Byte %d: CHANGING %d to %d (%s)\n", + node, if_num, rankx, i, + new_byte, bump_value, + (value_mask & + (1 << bump_value)) ? + "PBM" : "NOPBM"); + } + new_byte = bump_value; + } + } + + // last checks for count-related purposes + if (new_byte == best_byte && count_more > 0 && + count_less == 0) { + // we really should take best_byte + 1 + if (rl_print) { + debug("N%d.LMC%d.R%d: CADJMOR: Byte %d: CHANGING %d to %d\n", + node, if_num, rankx, i, + new_byte, best_byte + 1); + new_byte = best_byte + 1; + } + } else if ((new_byte < best_byte) && (count_same > 0)) { + // we really should take best_byte + if (rl_print) { + debug("N%d.LMC%d.R%d: CADJSAM: Byte %d: CHANGING %d to %d\n", + node, if_num, rankx, i, + new_byte, best_byte); + new_byte = best_byte; + } + } else if (new_byte > best_byte) { + if ((new_byte == (best_byte + 1)) && + count_more == 0 && count_less > 0) { + // we really should take best_byte + if (rl_print) { + debug("N%d.LMC%d.R%d: CADJLE1: Byte %d: CHANGING %d to %d\n", + node, if_num, rankx, i, + new_byte, best_byte); + new_byte = best_byte; + } + } else if ((new_byte >= (best_byte + 2)) && + ((count_more > 0) || (count_same > 0))) { + if (rl_print) { + debug("N%d.LMC%d.R%d: CADJLE2: Byte %d: CHANGING %d to %d\n", + node, if_num, rankx, i, + new_byte, best_byte + 1); + new_byte = best_byte + 1; + } + } + } + + if (rl_print) { + debug("N%d.LMC%d.R%d: SUMMARY: Byte %d: orig %d now %d, more %d same %d less %d, using %d\n", + node, if_num, rankx, i, orig_best_byte, + best_byte, count_more, count_same, count_less, + new_byte); + } + + // update the byte with the new value (NOTE: orig value in + // the CSR may not be current "best") + upd_rl_rank(&rl_rank, i, new_byte); + + // save new best for neighbor use + rank_best_bytes[i] = new_byte; + } /* for (i = 0; i < 8+ecc_ena; i++) */ + + ////////////////// this is the end of the BEST BYTE LOOP + + if (saved_rl_rank.u64 != rl_rank.u64) { + lmc_wr(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, if_num), + rl_rank.u64); + rl_rank.u64 = lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, if_num)); + debug("Adjusting Read-Leveling per-RANK settings.\n"); + } else { + debug("Not Adjusting Read-Leveling per-RANK settings.\n"); + } + display_rl_with_final(if_num, rl_rank, rankx); + + // FIXME: does this help make the output a little easier to focus? + if (rl_print > 0) + debug("-----------\n"); + +#if RLEXTRAS_PATCH +#define RLEVEL_RANKX_EXTRAS_INCR 0 + // if there are unused entries to be filled + if ((rank_mask & 0x0f) != 0x0f) { + // copy the current rank + union cvmx_lmcx_rlevel_rankx temp_rl_rank = rl_rank; + + if (rankx < 3) { +#if RLEVEL_RANKX_EXTRAS_INCR > 0 + int byte, delay; + + // modify the copy in prep for writing to empty slot(s) + for (byte = 0; byte < 9; byte++) { + delay = get_rl_rank(&temp_rl_rank, byte) + + RLEVEL_RANKX_EXTRAS_INCR; + if (delay > RLEVEL_BYTE_MSK) + delay = RLEVEL_BYTE_MSK; + upd_rl_rank(&temp_rl_rank, byte, delay); + } +#endif + + // if rank 0, write rank 1 and rank 2 here if empty + if (rankx == 0) { + // check that rank 1 is empty + if (!(rank_mask & (1 << 1))) { + debug("N%d.LMC%d.R%d: writing RLEVEL_RANK unused entry R%d.\n", + node, if_num, rankx, 1); + lmc_wr(priv, + CVMX_LMCX_RLEVEL_RANKX(1, + if_num), + temp_rl_rank.u64); + } + + // check that rank 2 is empty + if (!(rank_mask & (1 << 2))) { + debug("N%d.LMC%d.R%d: writing RLEVEL_RANK unused entry R%d.\n", + node, if_num, rankx, 2); + lmc_wr(priv, + CVMX_LMCX_RLEVEL_RANKX(2, + if_num), + temp_rl_rank.u64); + } + } + + // if ranks 0, 1 or 2, write rank 3 here if empty + // check that rank 3 is empty + if (!(rank_mask & (1 << 3))) { + debug("N%d.LMC%d.R%d: writing RLEVEL_RANK unused entry R%d.\n", + node, if_num, rankx, 3); + lmc_wr(priv, CVMX_LMCX_RLEVEL_RANKX(3, if_num), + temp_rl_rank.u64); + } + } + } +#endif /* RLEXTRAS_PATCH */ +} + +static void lmc_read_leveling(struct ddr_priv *priv) +{ + struct rl_score rl_score[RTT_NOM_OHMS_COUNT][RODT_OHMS_COUNT][4]; + union cvmx_lmcx_control ctl; + union cvmx_lmcx_config cfg; + int rankx; + char *s; + int i; + + /* + * 4.8.10 LMC Read Leveling + * + * LMC supports an automatic read-leveling separately per byte-lane + * using the DDR3 multipurpose register predefined pattern for system + * calibration defined in the JEDEC DDR3 specifications. + * + * All of DDR PLL, LMC CK, and LMC DRESET, and early LMC initializations + * must be completed prior to starting this LMC read-leveling sequence. + * + * Software could simply write the desired read-leveling values into + * LMC(0)_RLEVEL_RANK(0..3). This section describes a sequence that uses + * LMC's autoread-leveling capabilities. + * + * When LMC does the read-leveling sequence for a rank, it first enables + * the DDR3 multipurpose register predefined pattern for system + * calibration on the selected DRAM rank via a DDR3 MR3 write, then + * executes 64 RD operations at different internal delay settings, then + * disables the predefined pattern via another DDR3 MR3 write + * operation. LMC determines the pass or fail of each of the 64 settings + * independently for each byte lane, then writes appropriate + * LMC(0)_RLEVEL_RANK(0..3)[BYTE*] values for the rank. + * + * After read-leveling for a rank, software can read the 64 pass/fail + * indications for one byte lane via LMC(0)_RLEVEL_DBG[BITMASK]. + * Software can observe all pass/fail results for all byte lanes in a + * rank via separate read-leveling sequences on the rank with different + * LMC(0)_RLEVEL_CTL[BYTE] values. + * + * The 64 pass/fail results will typically have failures for the low + * delays, followed by a run of some passing settings, followed by more + * failures in the remaining high delays. LMC sets + * LMC(0)_RLEVEL_RANK(0..3)[BYTE*] to one of the passing settings. + * First, LMC selects the longest run of successes in the 64 results. + * (In the unlikely event that there is more than one longest run, LMC + * selects the first one.) Then if LMC(0)_RLEVEL_CTL[OFFSET_EN] = 1 and + * the selected run has more than LMC(0)_RLEVEL_CTL[OFFSET] successes, + * LMC selects the last passing setting in the run minus + * LMC(0)_RLEVEL_CTL[OFFSET]. Otherwise LMC selects the middle setting + * in the run (rounding earlier when necessary). We expect the + * read-leveling sequence to produce good results with the reset values + * LMC(0)_RLEVEL_CTL [OFFSET_EN]=1, LMC(0)_RLEVEL_CTL[OFFSET] = 2. + * + * The read-leveling sequence has the following steps: + * + * 1. Select desired LMC(0)_RLEVEL_CTL[OFFSET_EN,OFFSET,BYTE] settings. + * Do the remaining substeps 2-4 separately for each rank i with + * attached DRAM. + * + * 2. Without changing any other fields in LMC(0)_CONFIG, + * + * o write LMC(0)_SEQ_CTL[SEQ_SEL] to select read-leveling + * + * o write LMC(0)_CONFIG[RANKMASK] = (1 << i) + * + * o write LMC(0)_SEQ_CTL[INIT_START] = 1 + * + * This initiates the previously-described read-leveling. + * + * 3. Wait until LMC(0)_RLEVEL_RANKi[STATUS] != 2 + * + * LMC will have updated LMC(0)_RLEVEL_RANKi[BYTE*] for all byte + * lanes at this point. + * + * If ECC DRAM is not present (i.e. when DRAM is not attached to the + * DDR_CBS_0_* and DDR_CB<7:0> chip signals, or the DDR_DQS_<4>_* and + * DDR_DQ<35:32> chip signals), write LMC(0)_RLEVEL_RANK*[BYTE8] = + * LMC(0)_RLEVEL_RANK*[BYTE0]. Write LMC(0)_RLEVEL_RANK*[BYTE4] = + * LMC(0)_RLEVEL_RANK*[BYTE0]. + * + * 4. If desired, consult LMC(0)_RLEVEL_DBG[BITMASK] and compare to + * LMC(0)_RLEVEL_RANKi[BYTE*] for the lane selected by + * LMC(0)_RLEVEL_CTL[BYTE]. If desired, modify + * LMC(0)_RLEVEL_CTL[BYTE] to a new value and repeat so that all + * BITMASKs can be observed. + * + * 5. Initialize LMC(0)_RLEVEL_RANK* values for all unused ranks. + * + * Let rank i be a rank with attached DRAM. + * + * For all ranks j that do not have attached DRAM, set + * LMC(0)_RLEVEL_RANKj = LMC(0)_RLEVEL_RANKi. + * + * This read-leveling sequence can help select the proper CN70XX ODT + * resistance value (LMC(0)_COMP_CTL2[RODT_CTL]). A hardware-generated + * LMC(0)_RLEVEL_RANKi[BYTEj] value (for a used byte lane j) that is + * drastically different from a neighboring LMC(0)_RLEVEL_RANKi[BYTEk] + * (for a used byte lane k) can indicate that the CN70XX ODT value is + * bad. It is possible to simultaneously optimize both + * LMC(0)_COMP_CTL2[RODT_CTL] and LMC(0)_RLEVEL_RANKn[BYTE*] values by + * performing this read-leveling sequence for several + * LMC(0)_COMP_CTL2[RODT_CTL] values and selecting the one with the + * best LMC(0)_RLEVEL_RANKn[BYTE*] profile for the ranks. + */ + + rl_rodt_err = 0; + rl_dbg_loops = 1; + saved_int_zqcs_dis = 0; + max_adj_rl_del_inc = 0; + rl_print = RLEVEL_PRINTALL_DEFAULT; + +#ifdef ENABLE_HARDCODED_RLEVEL + part_number[21] = {0}; +#endif /* ENABLE_HARDCODED_RLEVEL */ + +#if PERFECT_BITMASK_COUNTING + pbm_lowsum_limit = 5; // FIXME: is this a good default? + // FIXME: PBM skip for RODT 240 and 34 + pbm_rodt_skip = (1U << ddr4_rodt_ctl_240_ohm) | + (1U << ddr4_rodt_ctl_34_ohm); +#endif /* PERFECT_BITMASK_COUNTING */ + + disable_rank_majority = 0; // control rank majority processing + + // default to mask 11b ODDs for DDR4 (except 73xx), else DISABLE + // for DDR3 + rldelay_bump_incr = 0; + disable_rlv_bump_this_byte = 0; + + enable_rldelay_bump = (ddr_type == DDR4_DRAM) ? + ((octeon_is_cpuid(OCTEON_CN73XX)) ? 1 : 3) : 0; + + s = lookup_env(priv, "ddr_disable_rank_majority"); + if (s) + disable_rank_majority = !!simple_strtoul(s, NULL, 0); + +#if PERFECT_BITMASK_COUNTING + s = lookup_env(priv, "ddr_pbm_lowsum_limit"); + if (s) + pbm_lowsum_limit = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_pbm_rodt_skip"); + if (s) + pbm_rodt_skip = simple_strtoul(s, NULL, 0); + memset(rank_perf, 0, sizeof(rank_perf)); +#endif /* PERFECT_BITMASK_COUNTING */ + + ctl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + save_ddr2t = ctl.cn78xx.ddr2t; + + cfg.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(if_num)); + ecc_ena = cfg.cn78xx.ecc_ena; + + s = lookup_env(priv, "ddr_rlevel_2t"); + if (s) + ctl.cn78xx.ddr2t = simple_strtoul(s, NULL, 0); + + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctl.u64); + + debug("LMC%d: Performing Read-Leveling\n", if_num); + + rl_ctl.u64 = lmc_rd(priv, CVMX_LMCX_RLEVEL_CTL(if_num)); + + rl_samples = c_cfg->rlevel_average_loops; + if (rl_samples == 0) { + rl_samples = RLEVEL_SAMPLES_DEFAULT; + // up the samples for these cases + if (dimm_count == 1 || num_ranks == 1) + rl_samples = rl_samples * 2 + 1; + } + + rl_compute = c_cfg->rlevel_compute; + rl_ctl.cn78xx.offset_en = c_cfg->offset_en; + rl_ctl.cn78xx.offset = spd_rdimm + ? c_cfg->offset_rdimm + : c_cfg->offset_udimm; + + int value = 1; // should ALWAYS be set + + s = lookup_env(priv, "ddr_rlevel_delay_unload"); + if (s) + value = !!simple_strtoul(s, NULL, 0); + rl_ctl.cn78xx.delay_unload_0 = value; + rl_ctl.cn78xx.delay_unload_1 = value; + rl_ctl.cn78xx.delay_unload_2 = value; + rl_ctl.cn78xx.delay_unload_3 = value; + + // use OR_DIS=1 to try for better results + rl_ctl.cn78xx.or_dis = 1; + + /* + * If we will be switching to 32bit mode level based on only + * four bits because there are only 4 ECC bits. + */ + rl_ctl.cn78xx.bitmask = (if_64b) ? 0xFF : 0x0F; + + // allow overrides + s = lookup_env(priv, "ddr_rlevel_ctl_or_dis"); + if (s) + rl_ctl.cn78xx.or_dis = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_ctl_bitmask"); + if (s) + rl_ctl.cn78xx.bitmask = simple_strtoul(s, NULL, 0); + + rl_comp_offs = spd_rdimm + ? c_cfg->rlevel_comp_offset_rdimm + : c_cfg->rlevel_comp_offset_udimm; + s = lookup_env(priv, "ddr_rlevel_comp_offset"); + if (s) + rl_comp_offs = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_offset"); + if (s) + rl_ctl.cn78xx.offset = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_offset_en"); + if (s) + rl_ctl.cn78xx.offset_en = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_ctl"); + if (s) + rl_ctl.u64 = simple_strtoul(s, NULL, 0); + + lmc_wr(priv, + CVMX_LMCX_RLEVEL_CTL(if_num), + rl_ctl.u64); + + // do this here so we can look at final RLEVEL_CTL[offset] setting... + s = lookup_env(priv, "ddr_enable_rldelay_bump"); + if (s) { + // also use as mask bits + enable_rldelay_bump = strtoul(s, NULL, 0); + } + + if (enable_rldelay_bump != 0) + rldelay_bump_incr = (rl_ctl.cn78xx.offset == 1) ? -1 : 1; + + s = lookup_env(priv, "ddr%d_rlevel_debug_loops", if_num); + if (s) + rl_dbg_loops = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rtt_nom_auto"); + if (s) + ddr_rtt_nom_auto = !!simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_average"); + if (s) + rl_samples = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_compute"); + if (s) + rl_compute = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_rlevel_printall"); + if (s) + rl_print = simple_strtoul(s, NULL, 0); + + debug("RLEVEL_CTL : 0x%016llx\n", + rl_ctl.u64); + debug("RLEVEL_OFFSET : %6d\n", + rl_ctl.cn78xx.offset); + debug("RLEVEL_OFFSET_EN : %6d\n", + rl_ctl.cn78xx.offset_en); + + /* + * The purpose for the indexed table is to sort the settings + * by the ohm value to simplify the testing when incrementing + * through the settings. (index => ohms) 1=120, 2=60, 3=40, + * 4=30, 5=20 + */ + min_rtt_nom_idx = (c_cfg->min_rtt_nom_idx == 0) ? + 1 : c_cfg->min_rtt_nom_idx; + max_rtt_nom_idx = (c_cfg->max_rtt_nom_idx == 0) ? + 5 : c_cfg->max_rtt_nom_idx; + + min_rodt_ctl = (c_cfg->min_rodt_ctl == 0) ? 1 : c_cfg->min_rodt_ctl; + max_rodt_ctl = (c_cfg->max_rodt_ctl == 0) ? 5 : c_cfg->max_rodt_ctl; + + s = lookup_env(priv, "ddr_min_rodt_ctl"); + if (s) + min_rodt_ctl = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_max_rodt_ctl"); + if (s) + max_rodt_ctl = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_min_rtt_nom_idx"); + if (s) + min_rtt_nom_idx = simple_strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_max_rtt_nom_idx"); + if (s) + max_rtt_nom_idx = simple_strtoul(s, NULL, 0); + +#ifdef ENABLE_HARDCODED_RLEVEL + if (c_cfg->rl_tbl) { + /* Check for hard-coded read-leveling settings */ + get_dimm_part_number(part_number, &dimm_config_table[0], + 0, ddr_type); + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + rl_rank.u64 = lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + + i = 0; + while (c_cfg->rl_tbl[i].part) { + debug("DIMM part number:"%s", SPD: "%s"\n", + c_cfg->rl_tbl[i].part, part_number); + if ((strcmp(part_number, + c_cfg->rl_tbl[i].part) == 0) && + (abs(c_cfg->rl_tbl[i].speed - + 2 * ddr_hertz / (1000 * 1000)) < 10)) { + debug("Using hard-coded read leveling for DIMM part number: "%s"\n", + part_number); + rl_rank.u64 = + c_cfg->rl_tbl[i].rl_rank[if_num][rankx]; + lmc_wr(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num), + rl_rank.u64); + rl_rank.u64 = + lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + display_rl(if_num, rl_rank, rankx); + /* Disable h/w read-leveling */ + rl_dbg_loops = 0; + break; + } + ++i; + } + } + } +#endif /* ENABLE_HARDCODED_RLEVEL */ + + max_adj_rl_del_inc = c_cfg->maximum_adjacent_rlevel_delay_increment; + s = lookup_env(priv, "ddr_maximum_adjacent_rlevel_delay_increment"); + if (s) + max_adj_rl_del_inc = strtoul(s, NULL, 0); + + while (rl_dbg_loops--) { + union cvmx_lmcx_modereg_params1 mp1; + union cvmx_lmcx_comp_ctl2 cc2; + + /* Initialize the error scoreboard */ + memset(rl_score, 0, sizeof(rl_score)); + + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + saved_ddr__ptune = cc2.cn78xx.ddr__ptune; + saved_ddr__ntune = cc2.cn78xx.ddr__ntune; + + /* Disable dynamic compensation settings */ + if (rl_comp_offs != 0) { + cc2.cn78xx.ptune = saved_ddr__ptune; + cc2.cn78xx.ntune = saved_ddr__ntune; + + /* + * Round up the ptune calculation to bias the odd + * cases toward ptune + */ + cc2.cn78xx.ptune += divide_roundup(rl_comp_offs, 2); + cc2.cn78xx.ntune -= rl_comp_offs / 2; + + ctl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + saved_int_zqcs_dis = ctl.s.int_zqcs_dis; + /* Disable ZQCS while in bypass. */ + ctl.s.int_zqcs_dis = 1; + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctl.u64); + + cc2.cn78xx.byp = 1; /* Enable bypass mode */ + lmc_wr(priv, CVMX_LMCX_COMP_CTL2(if_num), cc2.u64); + lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + /* Read again */ + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + debug("DDR__PTUNE/DDR__NTUNE : %d/%d\n", + cc2.cn78xx.ddr__ptune, cc2.cn78xx.ddr__ntune); + } + + mp1.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num)); + + for (rtt_idx = min_rtt_nom_idx; rtt_idx <= max_rtt_nom_idx; + ++rtt_idx) { + rtt_nom = imp_val->rtt_nom_table[rtt_idx]; + + /* + * When the read ODT mask is zero the dyn_rtt_nom_mask + * is zero than RTT_NOM will not be changing during + * read-leveling. Since the value is fixed we only need + * to test it once. + */ + if (dyn_rtt_nom_mask == 0) { + // flag not to print NOM ohms + print_nom_ohms = -1; + } else { + if (dyn_rtt_nom_mask & 1) + mp1.s.rtt_nom_00 = rtt_nom; + if (dyn_rtt_nom_mask & 2) + mp1.s.rtt_nom_01 = rtt_nom; + if (dyn_rtt_nom_mask & 4) + mp1.s.rtt_nom_10 = rtt_nom; + if (dyn_rtt_nom_mask & 8) + mp1.s.rtt_nom_11 = rtt_nom; + // FIXME? rank 0 ohms always? + print_nom_ohms = + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_00]; + } + + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num), + mp1.u64); + + if (print_nom_ohms >= 0 && rl_print > 1) { + debug("\n"); + debug("RTT_NOM %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_11], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_10], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_01], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_00], + mp1.s.rtt_nom_11, + mp1.s.rtt_nom_10, + mp1.s.rtt_nom_01, + mp1.s.rtt_nom_00); + } + + ddr_init_seq(priv, rank_mask, if_num); + + // Try RANK outside RODT to rearrange the output... + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + for (rodt_ctl = max_rodt_ctl; + rodt_ctl >= min_rodt_ctl; --rodt_ctl) + rodt_loop(priv, rankx, rl_score); + } + } + + /* Re-enable dynamic compensation settings. */ + if (rl_comp_offs != 0) { + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + + cc2.cn78xx.ptune = 0; + cc2.cn78xx.ntune = 0; + cc2.cn78xx.byp = 0; /* Disable bypass mode */ + lmc_wr(priv, CVMX_LMCX_COMP_CTL2(if_num), cc2.u64); + /* Read once */ + lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + + /* Read again */ + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + debug("DDR__PTUNE/DDR__NTUNE : %d/%d\n", + cc2.cn78xx.ddr__ptune, cc2.cn78xx.ddr__ntune); + + ctl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + /* Restore original setting */ + ctl.s.int_zqcs_dis = saved_int_zqcs_dis; + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctl.u64); + } + + int override_compensation = 0; + + s = lookup_env(priv, "ddr__ptune"); + if (s) + saved_ddr__ptune = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr__ntune"); + if (s) { + saved_ddr__ntune = strtoul(s, NULL, 0); + override_compensation = 1; + } + + if (override_compensation) { + cc2.cn78xx.ptune = saved_ddr__ptune; + cc2.cn78xx.ntune = saved_ddr__ntune; + + ctl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + saved_int_zqcs_dis = ctl.s.int_zqcs_dis; + /* Disable ZQCS while in bypass. */ + ctl.s.int_zqcs_dis = 1; + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctl.u64); + + cc2.cn78xx.byp = 1; /* Enable bypass mode */ + lmc_wr(priv, CVMX_LMCX_COMP_CTL2(if_num), cc2.u64); + /* Read again */ + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + + debug("DDR__PTUNE/DDR__NTUNE : %d/%d\n", + cc2.cn78xx.ptune, cc2.cn78xx.ntune); + } + + /* Evaluation block */ + /* Still at initial value? */ + int best_rodt_score = DEFAULT_BEST_RANK_SCORE; + int auto_rodt_ctl = 0; + int auto_rtt_nom = 0; + int rodt_score; + + rodt_row_skip_mask = 0; + + // just add specific RODT rows to the skip mask for DDR4 + // at this time... + if (ddr_type == DDR4_DRAM) { + // skip RODT row 34 ohms for all DDR4 types + rodt_row_skip_mask |= (1 << ddr4_rodt_ctl_34_ohm); + // skip RODT row 40 ohms for all DDR4 types + rodt_row_skip_mask |= (1 << ddr4_rodt_ctl_40_ohm); +#if ADD_48_OHM_SKIP + // skip RODT row 48 ohms for all DDR4 types + rodt_row_skip_mask |= (1 << ddr4_rodt_ctl_48_ohm); +#endif /* ADD_48OHM_SKIP */ +#if NOSKIP_40_48_OHM + // For now, do not skip RODT row 40 or 48 ohm when + // ddr_hertz is above 1075 MHz + if (ddr_hertz > 1075000000) { + // noskip RODT row 40 ohms + rodt_row_skip_mask &= + ~(1 << ddr4_rodt_ctl_40_ohm); + // noskip RODT row 48 ohms + rodt_row_skip_mask &= + ~(1 << ddr4_rodt_ctl_48_ohm); + } +#endif /* NOSKIP_40_48_OHM */ +#if NOSKIP_48_STACKED + // For now, do not skip RODT row 48 ohm for 2Rx4 + // stacked die DIMMs + if (is_stacked_die && num_ranks == 2 && + dram_width == 4) { + // noskip RODT row 48 ohms + rodt_row_skip_mask &= + ~(1 << ddr4_rodt_ctl_48_ohm); + } +#endif /* NOSKIP_48_STACKED */ +#if NOSKIP_FOR_MINI + // for now, leave all rows eligible when we have + // mini-DIMMs... + if (spd_dimm_type == 5 || spd_dimm_type == 6) + rodt_row_skip_mask = 0; +#endif /* NOSKIP_FOR_MINI */ +#if NOSKIP_FOR_2S_1R + // for now, leave all rows eligible when we have + // a 2-slot 1-rank config + if (dimm_count == 2 && num_ranks == 1) + rodt_row_skip_mask = 0; +#endif /* NOSKIP_FOR_2S_1R */ + + debug("Evaluating Read-Leveling Scoreboard for AUTO settings.\n"); + for (rtt_idx = min_rtt_nom_idx; + rtt_idx <= max_rtt_nom_idx; ++rtt_idx) { + rtt_nom = imp_val->rtt_nom_table[rtt_idx]; + + for (rodt_ctl = max_rodt_ctl; + rodt_ctl >= min_rodt_ctl; --rodt_ctl) { + rodt_score = 0; + for (rankx = 0; rankx < dimm_count * 4; + rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + debug("rl_score[rtt_nom=%d][rodt_ctl=%d][rankx=%d].score:%d\n", + rtt_nom, rodt_ctl, rankx, + rl_score[rtt_nom][rodt_ctl][rankx].score); + rodt_score += + rl_score[rtt_nom][rodt_ctl][rankx].score; + } + // FIXME: do we need to skip RODT rows + // here, like we do below in the + // by-RANK settings? + + /* + * When using automatic ODT settings use + * the ODT settings associated with the + * best score for all of the tested ODT + * combinations. + */ + + if (rodt_score < best_rodt_score || + (rodt_score == best_rodt_score && + (imp_val->rodt_ohms[rodt_ctl] > + imp_val->rodt_ohms[auto_rodt_ctl]))) { + debug("AUTO: new best score for rodt:%d (%d), new score:%d, previous score:%d\n", + rodt_ctl, + imp_val->rodt_ohms[rodt_ctl], + rodt_score, + best_rodt_score); + best_rodt_score = rodt_score; + auto_rodt_ctl = rodt_ctl; + auto_rtt_nom = rtt_nom; + } + } + } + + mp1.u64 = lmc_rd(priv, + CVMX_LMCX_MODEREG_PARAMS1(if_num)); + + if (ddr_rtt_nom_auto) { + /* Store the automatically set RTT_NOM value */ + if (dyn_rtt_nom_mask & 1) + mp1.s.rtt_nom_00 = auto_rtt_nom; + if (dyn_rtt_nom_mask & 2) + mp1.s.rtt_nom_01 = auto_rtt_nom; + if (dyn_rtt_nom_mask & 4) + mp1.s.rtt_nom_10 = auto_rtt_nom; + if (dyn_rtt_nom_mask & 8) + mp1.s.rtt_nom_11 = auto_rtt_nom; + } else { + /* + * restore the manual settings to the register + */ + mp1.s.rtt_nom_00 = default_rtt_nom[0]; + mp1.s.rtt_nom_01 = default_rtt_nom[1]; + mp1.s.rtt_nom_10 = default_rtt_nom[2]; + mp1.s.rtt_nom_11 = default_rtt_nom[3]; + } + + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS1(if_num), + mp1.u64); + debug("RTT_NOM %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_11], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_10], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_01], + imp_val->rtt_nom_ohms[mp1.s.rtt_nom_00], + mp1.s.rtt_nom_11, + mp1.s.rtt_nom_10, + mp1.s.rtt_nom_01, + mp1.s.rtt_nom_00); + + debug("RTT_WR %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 3)], + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 2)], + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 1)], + imp_val->rtt_wr_ohms[extr_wr(mp1.u64, 0)], + extr_wr(mp1.u64, 3), + extr_wr(mp1.u64, 2), + extr_wr(mp1.u64, 1), + extr_wr(mp1.u64, 0)); + + debug("DIC %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->dic_ohms[mp1.s.dic_11], + imp_val->dic_ohms[mp1.s.dic_10], + imp_val->dic_ohms[mp1.s.dic_01], + imp_val->dic_ohms[mp1.s.dic_00], + mp1.s.dic_11, + mp1.s.dic_10, + mp1.s.dic_01, + mp1.s.dic_00); + + if (ddr_type == DDR4_DRAM) { + union cvmx_lmcx_modereg_params2 mp2; + /* + * We must read the CSR, and not depend on + * odt_config[odt_idx].odt_mask2, since we could + * have overridden values with envvars. + * NOTE: this corrects the printout, since the + * CSR is not written with the old values... + */ + mp2.u64 = lmc_rd(priv, + CVMX_LMCX_MODEREG_PARAMS2(if_num)); + + debug("RTT_PARK %3d, %3d, %3d, %3d ohms : %x,%x,%x,%x\n", + imp_val->rtt_nom_ohms[mp2.s.rtt_park_11], + imp_val->rtt_nom_ohms[mp2.s.rtt_park_10], + imp_val->rtt_nom_ohms[mp2.s.rtt_park_01], + imp_val->rtt_nom_ohms[mp2.s.rtt_park_00], + mp2.s.rtt_park_11, + mp2.s.rtt_park_10, + mp2.s.rtt_park_01, + mp2.s.rtt_park_00); + + debug("%-45s : 0x%x,0x%x,0x%x,0x%x\n", + "VREF_RANGE", + mp2.s.vref_range_11, + mp2.s.vref_range_10, + mp2.s.vref_range_01, + mp2.s.vref_range_00); + + debug("%-45s : 0x%x,0x%x,0x%x,0x%x\n", + "VREF_VALUE", + mp2.s.vref_value_11, + mp2.s.vref_value_10, + mp2.s.vref_value_01, + mp2.s.vref_value_00); + } + + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + if (ddr_rodt_ctl_auto) { + cc2.cn78xx.rodt_ctl = auto_rodt_ctl; + } else { + // back to the original setting + cc2.cn78xx.rodt_ctl = default_rodt_ctl; + } + lmc_wr(priv, CVMX_LMCX_COMP_CTL2(if_num), cc2.u64); + cc2.u64 = lmc_rd(priv, CVMX_LMCX_COMP_CTL2(if_num)); + debug("Read ODT_CTL : 0x%x (%d ohms)\n", + cc2.cn78xx.rodt_ctl, + imp_val->rodt_ohms[cc2.cn78xx.rodt_ctl]); + + /* + * Use the delays associated with the best score for + * each individual rank + */ + debug("Evaluating Read-Leveling Scoreboard for per-RANK settings.\n"); + + // this is the the RANK MAJOR LOOP + for (rankx = 0; rankx < dimm_count * 4; rankx++) + rank_major_loop(priv, rankx, rl_score); + } /* Evaluation block */ + } /* while(rl_dbg_loops--) */ + + ctl.cn78xx.ddr2t = save_ddr2t; + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctl.u64); + ctl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + /* Display final 2T value */ + debug("DDR2T : %6d\n", + ctl.cn78xx.ddr2t); + + ddr_init_seq(priv, rank_mask, if_num); + + for (rankx = 0; rankx < dimm_count * 4; rankx++) { + u64 value; + int parameter_set = 0; + + if (!(rank_mask & (1 << rankx))) + continue; + + rl_rank.u64 = lmc_rd(priv, CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + + for (i = 0; i < 9; ++i) { + s = lookup_env(priv, "ddr%d_rlevel_rank%d_byte%d", + if_num, rankx, i); + if (s) { + parameter_set |= 1; + value = simple_strtoul(s, NULL, 0); + + upd_rl_rank(&rl_rank, i, value); + } + } + + s = lookup_env_ull(priv, "ddr%d_rlevel_rank%d", if_num, rankx); + if (s) { + parameter_set |= 1; + value = simple_strtoull(s, NULL, 0); + rl_rank.u64 = value; + } + + if (parameter_set) { + lmc_wr(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, if_num), + rl_rank.u64); + rl_rank.u64 = lmc_rd(priv, + CVMX_LMCX_RLEVEL_RANKX(rankx, + if_num)); + display_rl(if_num, rl_rank, rankx); + } + } +} + +int init_octeon3_ddr3_interface(struct ddr_priv *priv, + struct ddr_conf *_ddr_conf, u32 _ddr_hertz, + u32 cpu_hertz, u32 ddr_ref_hertz, int _if_num, + u32 _if_mask) +{ + union cvmx_lmcx_control ctrl; + int ret; + char *s; + int i; + + if_num = _if_num; + ddr_hertz = _ddr_hertz; + ddr_conf = _ddr_conf; + if_mask = _if_mask; + odt_1rank_config = ddr_conf->odt_1rank_config; + odt_2rank_config = ddr_conf->odt_2rank_config; + odt_4rank_config = ddr_conf->odt_4rank_config; + dimm_config_table = ddr_conf->dimm_config_table; + c_cfg = &ddr_conf->custom_lmc_config; + + /* + * Compute clock rates to the nearest picosecond. + */ + tclk_psecs = hertz_to_psecs(ddr_hertz); /* Clock in psecs */ + eclk_psecs = hertz_to_psecs(cpu_hertz); /* Clock in psecs */ + + dimm_count = 0; + /* Accumulate and report all the errors before giving up */ + fatal_error = 0; + + /* Flag that indicates safe DDR settings should be used */ + safe_ddr_flag = 0; + if_64b = 1; /* Octeon II Default: 64bit interface width */ + mem_size_mbytes = 0; + bank_bits = 0; + column_bits_start = 1; + use_ecc = 1; + min_cas_latency = 0, max_cas_latency = 0, override_cas_latency = 0; + spd_package = 0; + spd_rawcard = 0; + spd_rawcard_aorb = 0; + spd_rdimm_registers = 0; + is_stacked_die = 0; + is_3ds_dimm = 0; // 3DS + lranks_per_prank = 1; // 3DS: logical ranks per package rank + lranks_bits = 0; // 3DS: logical ranks bits + die_capacity = 0; // in Mbits; only used for 3DS + + wl_mask_err = 0; + dyn_rtt_nom_mask = 0; + ddr_disable_chip_reset = 1; + match_wl_rtt_nom = 0; + + internal_retries = 0; + + disable_deskew_training = 0; + restart_if_dsk_incomplete = 0; + last_lane = ((if_64b) ? 8 : 4) + use_ecc; + + disable_sequential_delay_check = 0; + wl_print = WLEVEL_PRINTALL_DEFAULT; + +#if ALLOW_BY_RANK_INIT + enable_by_rank_init = 1; // FIXME: default by-rank ON + saved_rank_mask = 0; +#endif /* ALLOW_BY_RANK_INIT */ + + node = 0; + +#if SWL_TRY_HWL_ALT + memset(hwl_alts, 0, sizeof(hwl_alts)); +#endif + + /* + * Initialize these to shut up the compiler. They are configured + * and used only for DDR4 + */ + ddr4_trrd_lmin = 6000; + ddr4_tccd_lmin = 6000; + + debug("\nInitializing node %d DDR interface %d, DDR Clock %d, DDR Reference Clock %d, CPUID 0x%08x\n", + node, if_num, ddr_hertz, ddr_ref_hertz, read_c0_prid()); + + if (dimm_config_table[0].spd_addrs[0] == 0 && + !dimm_config_table[0].spd_ptrs[0]) { + printf("ERROR: No dimms specified in the dimm_config_table.\n"); + return -1; + } + + // allow some overrides to be done + + // this one controls several things related to DIMM geometry: HWL and RL + disable_sequential_delay_check = c_cfg->disable_sequential_delay_check; + s = lookup_env(priv, "ddr_disable_sequential_delay_check"); + if (s) + disable_sequential_delay_check = strtoul(s, NULL, 0); + + // this one controls whether chip RESET is done, or LMC init restarted + // from step 6.9.6 + s = lookup_env(priv, "ddr_disable_chip_reset"); + if (s) + ddr_disable_chip_reset = !!strtoul(s, NULL, 0); + + // this one controls whether Deskew Training is performed + s = lookup_env(priv, "ddr_disable_deskew_training"); + if (s) + disable_deskew_training = !!strtoul(s, NULL, 0); + + if (ddr_verbose(priv)) { + printf("DDR SPD Table:"); + for (didx = 0; didx < DDR_CFG_T_MAX_DIMMS; ++didx) { + if (dimm_config_table[didx].spd_addrs[0] == 0) + break; + + printf(" --ddr%dspd=0x%02x", if_num, + dimm_config_table[didx].spd_addrs[0]); + if (dimm_config_table[didx].spd_addrs[1] != 0) + printf(",0x%02x", + dimm_config_table[didx].spd_addrs[1]); + } + printf("\n"); + } + + /* + * Walk the DRAM Socket Configuration Table to see what is installed. + */ + for (didx = 0; didx < DDR_CFG_T_MAX_DIMMS; ++didx) { + /* Check for lower DIMM socket populated */ + if (validate_dimm(priv, &dimm_config_table[didx], 0)) { + if (ddr_verbose(priv)) + report_dimm(&dimm_config_table[didx], 0, + dimm_count, if_num); + ++dimm_count; + } else { + break; + } /* Finished when there is no lower DIMM */ + } + + initialize_ddr_clock(priv, ddr_conf, cpu_hertz, ddr_hertz, + ddr_ref_hertz, if_num, if_mask); + + if (!odt_1rank_config) + odt_1rank_config = disable_odt_config; + if (!odt_2rank_config) + odt_2rank_config = disable_odt_config; + if (!odt_4rank_config) + odt_4rank_config = disable_odt_config; + + s = env_get("ddr_safe"); + if (s) { + safe_ddr_flag = !!simple_strtoul(s, NULL, 0); + printf("Parameter found in environment. ddr_safe = %d\n", + safe_ddr_flag); + } + + if (dimm_count == 0) { + printf("ERROR: DIMM 0 not detected.\n"); + return (-1); + } + + if (c_cfg->mode32b) + if_64b = 0; + + s = lookup_env(priv, "if_64b"); + if (s) + if_64b = !!simple_strtoul(s, NULL, 0); + + if (if_64b == 1) { + if (octeon_is_cpuid(OCTEON_CN70XX)) { + printf("64-bit interface width is not supported for this Octeon model\n"); + ++fatal_error; + } + } + + /* ddr_type only indicates DDR4 or DDR3 */ + ddr_type = (read_spd(&dimm_config_table[0], 0, + DDR4_SPD_KEY_BYTE_DEVICE_TYPE) == 0x0C) ? 4 : 3; + debug("DRAM Device Type: DDR%d\n", ddr_type); + + if (ddr_type == DDR4_DRAM) { + int spd_module_type; + int asymmetric; + const char *signal_load[4] = { "", "MLS", "3DS", "RSV" }; + + imp_val = &ddr4_impedence_val; + + spd_addr = + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_ADDRESSING_ROW_COL_BITS); + spd_org = + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MODULE_ORGANIZATION); + spd_banks = + 0xFF & read_spd(&dimm_config_table[0], 0, + DDR4_SPD_DENSITY_BANKS); + + bank_bits = + (2 + ((spd_banks >> 4) & 0x3)) + ((spd_banks >> 6) & 0x3); + /* Controller can only address 4 bits. */ + bank_bits = min((int)bank_bits, 4); + + spd_package = + 0XFF & read_spd(&dimm_config_table[0], 0, + DDR4_SPD_PACKAGE_TYPE); + if (spd_package & 0x80) { // non-monolithic device + is_stacked_die = ((spd_package & 0x73) == 0x11); + debug("DDR4: Package Type 0x%02x (%s), %d die\n", + spd_package, signal_load[(spd_package & 3)], + ((spd_package >> 4) & 7) + 1); + is_3ds_dimm = ((spd_package & 3) == 2); // is it 3DS? + if (is_3ds_dimm) { // is it 3DS? + lranks_per_prank = ((spd_package >> 4) & 7) + 1; + // FIXME: should make sure it is only 2H or 4H + // or 8H? + lranks_bits = lranks_per_prank >> 1; + if (lranks_bits == 4) + lranks_bits = 3; + } + } else if (spd_package != 0) { + // FIXME: print non-zero monolithic device definition + debug("DDR4: Package Type MONOLITHIC: %d die, signal load %d\n", + ((spd_package >> 4) & 7) + 1, (spd_package & 3)); + } + + asymmetric = (spd_org >> 6) & 1; + if (asymmetric) { + int spd_secondary_pkg = + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_SECONDARY_PACKAGE_TYPE); + debug("DDR4: Module Organization: ASYMMETRICAL: Secondary Package Type 0x%02x\n", + spd_secondary_pkg); + } else { + u64 bus_width = + 8 << (0x07 & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MODULE_MEMORY_BUS_WIDTH)); + u64 ddr_width = 4 << ((spd_org >> 0) & 0x7); + u64 module_cap; + int shift = (spd_banks & 0x0F); + + die_capacity = (shift < 8) ? (256UL << shift) : + ((12UL << (shift & 1)) << 10); + debug("DDR4: Module Organization: SYMMETRICAL: capacity per die %d %cbit\n", + (die_capacity > 512) ? (die_capacity >> 10) : + die_capacity, (die_capacity > 512) ? 'G' : 'M'); + module_cap = ((u64)die_capacity << 20) / 8UL * + bus_width / ddr_width * + (1UL + ((spd_org >> 3) & 0x7)); + + // is it 3DS? + if (is_3ds_dimm) { + module_cap *= (u64)(((spd_package >> 4) & 7) + + 1); + } + debug("DDR4: Module Organization: SYMMETRICAL: capacity per module %lld GB\n", + module_cap >> 30); + } + + spd_rawcard = + 0xFF & read_spd(&dimm_config_table[0], 0, + DDR4_SPD_REFERENCE_RAW_CARD); + debug("DDR4: Reference Raw Card 0x%02x\n", spd_rawcard); + + spd_module_type = + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_KEY_BYTE_MODULE_TYPE); + if (spd_module_type & 0x80) { // HYBRID module + debug("DDR4: HYBRID module, type %s\n", + ((spd_module_type & 0x70) == + 0x10) ? "NVDIMM" : "UNKNOWN"); + } + spd_thermal_sensor = + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MODULE_THERMAL_SENSOR); + spd_dimm_type = spd_module_type & 0x0F; + spd_rdimm = (spd_dimm_type == 1) || (spd_dimm_type == 5) || + (spd_dimm_type == 8); + if (spd_rdimm) { + u16 spd_mfgr_id, spd_register_rev, spd_mod_attr; + static const u16 manu_ids[4] = { + 0xb380, 0x3286, 0x9780, 0xb304 + }; + static const char *manu_names[4] = { + "XXX", "XXXXXXX", "XX", "XXXXX" + }; + int mc; + + spd_mfgr_id = + (0xFFU & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_REGISTER_MANUFACTURER_ID_LSB)) | + ((0xFFU & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_REGISTER_MANUFACTURER_ID_MSB)) + << 8); + spd_register_rev = + 0xFFU & read_spd(&dimm_config_table[0], 0, + DDR4_SPD_REGISTER_REVISION_NUMBER); + for (mc = 0; mc < 4; mc++) + if (manu_ids[mc] == spd_mfgr_id) + break; + + debug("DDR4: RDIMM Register Manufacturer ID: %s, Revision: 0x%02x\n", + (mc >= 4) ? "UNKNOWN" : manu_names[mc], + spd_register_rev); + + // RAWCARD A or B must be bit 7=0 and bits 4-0 + // either 00000(A) or 00001(B) + spd_rawcard_aorb = ((spd_rawcard & 0x9fUL) <= 1); + // RDIMM Module Attributes + spd_mod_attr = + 0xFFU & read_spd(&dimm_config_table[0], 0, + DDR4_SPD_UDIMM_ADDR_MAPPING_FROM_EDGE); + spd_rdimm_registers = ((1 << (spd_mod_attr & 3)) >> 1); + debug("DDR4: RDIMM Module Attributes (0x%02x): Register Type DDR4RCD%02d, DRAM rows %d, Registers %d\n", + spd_mod_attr, (spd_mod_attr >> 4) + 1, + ((1 << ((spd_mod_attr >> 2) & 3)) >> 1), + spd_rdimm_registers); + } + dimm_type_name = ddr4_dimm_types[spd_dimm_type]; + } else { /* if (ddr_type == DDR4_DRAM) */ + const char *signal_load[4] = { "UNK", "MLS", "SLS", "RSV" }; + + imp_val = &ddr3_impedence_val; + + spd_addr = + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_ADDRESSING_ROW_COL_BITS); + spd_org = + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MODULE_ORGANIZATION); + spd_banks = + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_DENSITY_BANKS) & 0xff; + + bank_bits = 3 + ((spd_banks >> 4) & 0x7); + /* Controller can only address 3 bits. */ + bank_bits = min((int)bank_bits, 3); + spd_dimm_type = + 0x0f & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_KEY_BYTE_MODULE_TYPE); + spd_rdimm = (spd_dimm_type == 1) || (spd_dimm_type == 5) || + (spd_dimm_type == 9); + + spd_package = + 0xFF & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_SDRAM_DEVICE_TYPE); + if (spd_package & 0x80) { // non-standard device + debug("DDR3: Device Type 0x%02x (%s), %d die\n", + spd_package, signal_load[(spd_package & 3)], + ((1 << ((spd_package >> 4) & 7)) >> 1)); + } else if (spd_package != 0) { + // FIXME: print non-zero monolithic device definition + debug("DDR3: Device Type MONOLITHIC: %d die, signal load %d\n", + ((1 << (spd_package >> 4) & 7) >> 1), + (spd_package & 3)); + } + + spd_rawcard = + 0xFF & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_REFERENCE_RAW_CARD); + debug("DDR3: Reference Raw Card 0x%02x\n", spd_rawcard); + spd_thermal_sensor = + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MODULE_THERMAL_SENSOR); + + if (spd_rdimm) { + int spd_mfgr_id, spd_register_rev, spd_mod_attr; + + spd_mfgr_id = + (0xFFU & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_REGISTER_MANUFACTURER_ID_LSB)) | + ((0xFFU & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_REGISTER_MANUFACTURER_ID_MSB)) + << 8); + spd_register_rev = + 0xFFU & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_REGISTER_REVISION_NUMBER); + debug("DDR3: RDIMM Register Manufacturer ID 0x%x Revision 0x%02x\n", + spd_mfgr_id, spd_register_rev); + // Module Attributes + spd_mod_attr = + 0xFFU & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_ADDRESS_MAPPING); + spd_rdimm_registers = ((1 << (spd_mod_attr & 3)) >> 1); + debug("DDR3: RDIMM Module Attributes (0x%02x): DRAM rows %d, Registers %d\n", + spd_mod_attr, + ((1 << ((spd_mod_attr >> 2) & 3)) >> 1), + spd_rdimm_registers); + } + dimm_type_name = ddr3_dimm_types[spd_dimm_type]; + } + + if (spd_thermal_sensor & 0x80) { + debug("DDR%d: SPD: Thermal Sensor PRESENT\n", + (ddr_type == DDR4_DRAM) ? 4 : 3); + } + + debug("spd_addr : %#06x\n", spd_addr); + debug("spd_org : %#06x\n", spd_org); + debug("spd_banks : %#06x\n", spd_banks); + + row_bits = 12 + ((spd_addr >> 3) & 0x7); + col_bits = 9 + ((spd_addr >> 0) & 0x7); + + num_ranks = 1 + ((spd_org >> 3) & 0x7); + dram_width = 4 << ((spd_org >> 0) & 0x7); + num_banks = 1 << bank_bits; + + s = lookup_env(priv, "ddr_num_ranks"); + if (s) + num_ranks = simple_strtoul(s, NULL, 0); + +#if ALLOW_BY_RANK_INIT + s = lookup_env(priv, "ddr_enable_by_rank_init"); + if (s) + enable_by_rank_init = !!simple_strtoul(s, NULL, 0); + + // FIXME: for now, we can only handle a DDR4 2rank-1slot config + // FIXME: also, by-rank init does not work correctly if 32-bit mode... + if (enable_by_rank_init && (ddr_type != DDR4_DRAM || + dimm_count != 1 || if_64b != 1 || + num_ranks != 2)) + enable_by_rank_init = 0; + + if (enable_by_rank_init) { + struct dimm_odt_config *odt_config; + union cvmx_lmcx_modereg_params1 mp1; + union cvmx_lmcx_modereg_params2 modereg_params2; + int by_rank_rodt, by_rank_wr, by_rank_park; + + // Do ODT settings changes which work best for 2R-1S configs + debug("DDR4: 2R-1S special BY-RANK init ODT settings updated\n"); + + // setup for modifying config table values - 2 ranks and 1 DIMM + odt_config = + (struct dimm_odt_config *)&ddr_conf->odt_2rank_config[0]; + + // original was 80, first try was 60 + by_rank_rodt = ddr4_rodt_ctl_48_ohm; + s = lookup_env(priv, "ddr_by_rank_rodt"); + if (s) + by_rank_rodt = strtoul(s, NULL, 0); + + odt_config->qs_dic = /*RODT_CTL */ by_rank_rodt; + + // this is for MODEREG_PARAMS1 fields + // fetch the original settings + mp1.u64 = odt_config->modereg_params1.u64; + + by_rank_wr = ddr4_rttwr_80ohm; // originals were 240 + s = lookup_env(priv, "ddr_by_rank_wr"); + if (s) + by_rank_wr = simple_strtoul(s, NULL, 0); + + // change specific settings here... + insrt_wr(&mp1.u64, /*rank */ 00, by_rank_wr); + insrt_wr(&mp1.u64, /*rank */ 01, by_rank_wr); + + // save final settings + odt_config->modereg_params1.u64 = mp1.u64; + + // this is for MODEREG_PARAMS2 fields + // fetch the original settings + modereg_params2.u64 = odt_config->modereg_params2.u64; + + by_rank_park = ddr4_rttpark_none; // originals were 120 + s = lookup_env(priv, "ddr_by_rank_park"); + if (s) + by_rank_park = simple_strtoul(s, NULL, 0); + + // change specific settings here... + modereg_params2.s.rtt_park_00 = by_rank_park; + modereg_params2.s.rtt_park_01 = by_rank_park; + + // save final settings + odt_config->modereg_params2.u64 = modereg_params2.u64; + } +#endif /* ALLOW_BY_RANK_INIT */ + + /* + * FIX + * Check that values are within some theoretical limits. + * col_bits(min) = row_lsb(min) - bank_bits(max) - bus_bits(max) = + * 14 - 3 - 4 = 7 + * col_bits(max) = row_lsb(max) - bank_bits(min) - bus_bits(min) = + * 18 - 2 - 3 = 13 + */ + if (col_bits > 13 || col_bits < 7) { + printf("Unsupported number of Col Bits: %d\n", col_bits); + ++fatal_error; + } + + /* + * FIX + * Check that values are within some theoretical limits. + * row_bits(min) = pbank_lsb(min) - row_lsb(max) - rank_bits = + * 26 - 18 - 1 = 7 + * row_bits(max) = pbank_lsb(max) - row_lsb(min) - rank_bits = + * 33 - 14 - 1 = 18 + */ + if (row_bits > 18 || row_bits < 7) { + printf("Unsupported number of Row Bits: %d\n", row_bits); + ++fatal_error; + } + + s = lookup_env(priv, "ddr_rdimm_ena"); + if (s) + spd_rdimm = !!simple_strtoul(s, NULL, 0); + + wl_loops = WLEVEL_LOOPS_DEFAULT; + // accept generic or interface-specific override + s = lookup_env(priv, "ddr_wlevel_loops"); + if (!s) + s = lookup_env(priv, "ddr%d_wlevel_loops", if_num); + + if (s) + wl_loops = strtoul(s, NULL, 0); + + s = lookup_env(priv, "ddr_ranks"); + if (s) + num_ranks = simple_strtoul(s, NULL, 0); + + bunk_enable = (num_ranks > 1); + + if (octeon_is_cpuid(OCTEON_CN7XXX)) + column_bits_start = 3; + else + printf("ERROR: Unsupported Octeon model: 0x%x\n", + read_c0_prid()); + + row_lsb = column_bits_start + col_bits + bank_bits - (!if_64b); + debug("row_lsb = column_bits_start + col_bits + bank_bits = %d\n", + row_lsb); + + pbank_lsb = row_lsb + row_bits + bunk_enable; + debug("pbank_lsb = row_lsb + row_bits + bunk_enable = %d\n", pbank_lsb); + + if (lranks_per_prank > 1) { + pbank_lsb = row_lsb + row_bits + lranks_bits + bunk_enable; + debug("DDR4: 3DS: pbank_lsb = (%d row_lsb) + (%d row_bits) + (%d lranks_bits) + (%d bunk_enable) = %d\n", + row_lsb, row_bits, lranks_bits, bunk_enable, pbank_lsb); + } + + mem_size_mbytes = dimm_count * ((1ull << pbank_lsb) >> 20); + if (num_ranks == 4) { + /* + * Quad rank dimm capacity is equivalent to two dual-rank + * dimms. + */ + mem_size_mbytes *= 2; + } + + /* + * Mask with 1 bits set for for each active rank, allowing 2 bits + * per dimm. This makes later calculations simpler, as a variety + * of CSRs use this layout. This init needs to be updated for dual + * configs (ie non-identical DIMMs). + * + * Bit 0 = dimm0, rank 0 + * Bit 1 = dimm0, rank 1 + * Bit 2 = dimm1, rank 0 + * Bit 3 = dimm1, rank 1 + * ... + */ + rank_mask = 0x1; + if (num_ranks > 1) + rank_mask = 0x3; + if (num_ranks > 2) + rank_mask = 0xf; + + for (i = 1; i < dimm_count; i++) + rank_mask |= ((rank_mask & 0x3) << (2 * i)); + + /* + * If we are booting from RAM, the DRAM controller is + * already set up. Just return the memory size + */ + if (priv->flags & FLAG_RAM_RESIDENT) { + debug("Ram Boot: Skipping LMC config\n"); + return mem_size_mbytes; + } + + if (ddr_type == DDR4_DRAM) { + spd_ecc = + !!(read_spd + (&dimm_config_table[0], 0, + DDR4_SPD_MODULE_MEMORY_BUS_WIDTH) & 8); + } else { + spd_ecc = + !!(read_spd + (&dimm_config_table[0], 0, + DDR3_SPD_MEMORY_BUS_WIDTH) & 8); + } + + char rank_spec[8]; + + printable_rank_spec(rank_spec, num_ranks, dram_width, spd_package); + debug("Summary: %d %s%s %s %s, row bits=%d, col bits=%d, bank bits=%d\n", + dimm_count, dimm_type_name, (dimm_count > 1) ? "s" : "", + rank_spec, + (spd_ecc) ? "ECC" : "non-ECC", row_bits, col_bits, bank_bits); + + if (ddr_type == DDR4_DRAM) { + spd_cas_latency = + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_CAS_LATENCIES_BYTE0)) << 0); + spd_cas_latency |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_CAS_LATENCIES_BYTE1)) << 8); + spd_cas_latency |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_CAS_LATENCIES_BYTE2)) << 16); + spd_cas_latency |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR4_SPD_CAS_LATENCIES_BYTE3)) << 24); + } else { + spd_cas_latency = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_CAS_LATENCIES_LSB); + spd_cas_latency |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_CAS_LATENCIES_MSB)) << 8); + } + debug("spd_cas_latency : %#06x\n", spd_cas_latency); + + if (ddr_type == DDR4_DRAM) { + /* + * No other values for DDR4 MTB and FTB are specified at the + * current time so don't bother reading them. Can't speculate + * how new values will be represented. + */ + int spdmtb = 125; + int spdftb = 1; + + taamin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_CAS_LATENCY_TAAMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], + 0, DDR4_SPD_MIN_CAS_LATENCY_FINE_TAAMIN); + + ddr4_tckavgmin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MINIMUM_CYCLE_TIME_TCKAVGMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_CYCLE_TIME_FINE_TCKAVGMIN); + + ddr4_tckavgmax = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MAXIMUM_CYCLE_TIME_TCKAVGMAX) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MAX_CYCLE_TIME_FINE_TCKAVGMAX); + + ddr4_trdcmin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_RAS_CAS_DELAY_TRCDMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_RAS_TO_CAS_DELAY_FINE_TRCDMIN); + + ddr4_trpmin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_ROW_PRECHARGE_DELAY_TRPMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_ROW_PRECHARGE_DELAY_FINE_TRPMIN); + + ddr4_trasmin = spdmtb * + (((read_spd + (&dimm_config_table[0], 0, + DDR4_SPD_UPPER_NIBBLES_TRAS_TRC) & 0xf) << 8) + + (read_spd + (&dimm_config_table[0], 0, + DDR4_SPD_MIN_ACTIVE_PRECHARGE_LSB_TRASMIN) & 0xff)); + + ddr4_trcmin = spdmtb * + ((((read_spd + (&dimm_config_table[0], 0, + DDR4_SPD_UPPER_NIBBLES_TRAS_TRC) >> 4) & 0xf) << + 8) + (read_spd + (&dimm_config_table[0], 0, + DDR4_SPD_MIN_ACTIVE_REFRESH_LSB_TRCMIN) & + 0xff)) + + spdftb * (signed char)read_spd(&dimm_config_table[0], + 0, + DDR4_SPD_MIN_ACT_TO_ACT_REFRESH_DELAY_FINE_TRCMIN); + + ddr4_trfc1min = spdmtb * (((read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC1MIN) & 0xff) << + 8) + (read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC1MIN) & 0xff)); + + ddr4_trfc2min = spdmtb * (((read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC2MIN) & 0xff) << + 8) + (read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC2MIN) & 0xff)); + + ddr4_trfc4min = spdmtb * (((read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_REFRESH_RECOVERY_MSB_TRFC4MIN) & 0xff) << + 8) + (read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_REFRESH_RECOVERY_LSB_TRFC4MIN) & 0xff)); + + ddr4_tfawmin = spdmtb * (((read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_FOUR_ACTIVE_WINDOW_MSN_TFAWMIN) & 0xf) << + 8) + (read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_FOUR_ACTIVE_WINDOW_LSB_TFAWMIN) & 0xff)); + + ddr4_trrd_smin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_ROW_ACTIVE_DELAY_SAME_TRRD_SMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_ACT_TO_ACT_DELAY_DIFF_FINE_TRRD_SMIN); + + ddr4_trrd_lmin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_ROW_ACTIVE_DELAY_DIFF_TRRD_LMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_ACT_TO_ACT_DELAY_SAME_FINE_TRRD_LMIN); + + ddr4_tccd_lmin = spdmtb * read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_CAS_TO_CAS_DELAY_TCCD_LMIN) + + spdftb * (signed char)read_spd(&dimm_config_table[0], 0, + DDR4_SPD_MIN_CAS_TO_CAS_DELAY_FINE_TCCD_LMIN); + + debug("%-45s : %6d ps\n", "Medium Timebase (MTB)", spdmtb); + debug("%-45s : %6d ps\n", "Fine Timebase (FTB)", spdftb); + + debug("%-45s : %6d ps (%ld MT/s)\n", + "SDRAM Minimum Cycle Time (tCKAVGmin)", ddr4_tckavgmin, + pretty_psecs_to_mts(ddr4_tckavgmin)); + debug("%-45s : %6d ps\n", + "SDRAM Maximum Cycle Time (tCKAVGmax)", ddr4_tckavgmax); + debug("%-45s : %6d ps\n", "Minimum CAS Latency Time (taamin)", + taamin); + debug("%-45s : %6d ps\n", + "Minimum RAS to CAS Delay Time (tRCDmin)", ddr4_trdcmin); + debug("%-45s : %6d ps\n", + "Minimum Row Precharge Delay Time (tRPmin)", ddr4_trpmin); + debug("%-45s : %6d ps\n", + "Minimum Active to Precharge Delay (tRASmin)", + ddr4_trasmin); + debug("%-45s : %6d ps\n", + "Minimum Active to Active/Refr. Delay (tRCmin)", + ddr4_trcmin); + debug("%-45s : %6d ps\n", + "Minimum Refresh Recovery Delay (tRFC1min)", + ddr4_trfc1min); + debug("%-45s : %6d ps\n", + "Minimum Refresh Recovery Delay (tRFC2min)", + ddr4_trfc2min); + debug("%-45s : %6d ps\n", + "Minimum Refresh Recovery Delay (tRFC4min)", + ddr4_trfc4min); + debug("%-45s : %6d ps\n", + "Minimum Four Activate Window Time (tFAWmin)", + ddr4_tfawmin); + debug("%-45s : %6d ps\n", + "Minimum Act. to Act. Delay (tRRD_Smin)", ddr4_trrd_smin); + debug("%-45s : %6d ps\n", + "Minimum Act. to Act. Delay (tRRD_Lmin)", ddr4_trrd_lmin); + debug("%-45s : %6d ps\n", + "Minimum CAS to CAS Delay Time (tCCD_Lmin)", + ddr4_tccd_lmin); + +#define DDR4_TWR 15000 +#define DDR4_TWTR_S 2500 + + tckmin = ddr4_tckavgmin; + twr = DDR4_TWR; + trcd = ddr4_trdcmin; + trrd = ddr4_trrd_smin; + trp = ddr4_trpmin; + tras = ddr4_trasmin; + trc = ddr4_trcmin; + trfc = ddr4_trfc1min; + twtr = DDR4_TWTR_S; + tfaw = ddr4_tfawmin; + + if (spd_rdimm) { + spd_addr_mirror = read_spd(&dimm_config_table[0], 0, + DDR4_SPD_RDIMM_ADDR_MAPPING_FROM_REGISTER_TO_DRAM) & + 0x1; + } else { + spd_addr_mirror = read_spd(&dimm_config_table[0], 0, + DDR4_SPD_UDIMM_ADDR_MAPPING_FROM_EDGE) & 0x1; + } + debug("spd_addr_mirror : %#06x\n", spd_addr_mirror); + } else { + spd_mtb_dividend = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MEDIUM_TIMEBASE_DIVIDEND); + spd_mtb_divisor = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MEDIUM_TIMEBASE_DIVISOR); + spd_tck_min = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MINIMUM_CYCLE_TIME_TCKMIN); + spd_taa_min = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_CAS_LATENCY_TAAMIN); + + spd_twr = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_WRITE_RECOVERY_TWRMIN); + spd_trcd = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_RAS_CAS_DELAY_TRCDMIN); + spd_trrd = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_ROW_ACTIVE_DELAY_TRRDMIN); + spd_trp = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_ROW_PRECHARGE_DELAY_TRPMIN); + spd_tras = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_ACTIVE_PRECHARGE_LSB_TRASMIN); + spd_tras |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_UPPER_NIBBLES_TRAS_TRC) & 0xf) << 8); + spd_trc = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_ACTIVE_REFRESH_LSB_TRCMIN); + spd_trc |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_UPPER_NIBBLES_TRAS_TRC) & 0xf0) << 4); + spd_trfc = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_REFRESH_RECOVERY_LSB_TRFCMIN); + spd_trfc |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_REFRESH_RECOVERY_MSB_TRFCMIN)) << + 8); + spd_twtr = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_INTERNAL_WRITE_READ_CMD_TWTRMIN); + spd_trtp = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_INTERNAL_READ_PRECHARGE_CMD_TRTPMIN); + spd_tfaw = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_MIN_FOUR_ACTIVE_WINDOW_TFAWMIN); + spd_tfaw |= + ((0xff & + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_UPPER_NIBBLE_TFAW) & 0xf) << 8); + spd_addr_mirror = + 0xff & read_spd(&dimm_config_table[0], 0, + DDR3_SPD_ADDRESS_MAPPING) & 0x1; + /* Only address mirror unbuffered dimms. */ + spd_addr_mirror = spd_addr_mirror && !spd_rdimm; + ftb_dividend = + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_FINE_TIMEBASE_DIVIDEND_DIVISOR) >> 4; + ftb_divisor = + read_spd(&dimm_config_table[0], 0, + DDR3_SPD_FINE_TIMEBASE_DIVIDEND_DIVISOR) & 0xf; + /* Make sure that it is not 0 */ + ftb_divisor = (ftb_divisor == 0) ? 1 : ftb_divisor; + + debug("spd_twr : %#06x\n", spd_twr); + debug("spd_trcd : %#06x\n", spd_trcd); + debug("spd_trrd : %#06x\n", spd_trrd); + debug("spd_trp : %#06x\n", spd_trp); + debug("spd_tras : %#06x\n", spd_tras); + debug("spd_trc : %#06x\n", spd_trc); + debug("spd_trfc : %#06x\n", spd_trfc); + debug("spd_twtr : %#06x\n", spd_twtr); + debug("spd_trtp : %#06x\n", spd_trtp); + debug("spd_tfaw : %#06x\n", spd_tfaw); + debug("spd_addr_mirror : %#06x\n", spd_addr_mirror); + + mtb_psec = spd_mtb_dividend * 1000 / spd_mtb_divisor; + taamin = mtb_psec * spd_taa_min; + taamin += ftb_dividend * + (signed char)read_spd(&dimm_config_table[0], + 0, DDR3_SPD_MIN_CAS_LATENCY_FINE_TAAMIN) / + ftb_divisor; + tckmin = mtb_psec * spd_tck_min; + tckmin += ftb_dividend * + (signed char)read_spd(&dimm_config_table[0], + 0, DDR3_SPD_MINIMUM_CYCLE_TIME_FINE_TCKMIN) / + ftb_divisor; + + twr = spd_twr * mtb_psec; + trcd = spd_trcd * mtb_psec; + trrd = spd_trrd * mtb_psec; + trp = spd_trp * mtb_psec; + tras = spd_tras * mtb_psec; + trc = spd_trc * mtb_psec; + trfc = spd_trfc * mtb_psec; + if (octeon_is_cpuid(OCTEON_CN78XX_PASS2_X) && trfc < 260000) { + // default to this - because it works... + int new_trfc = 260000; + + s = env_get("ddr_trfc"); + if (s) { + new_trfc = simple_strtoul(s, NULL, 0); + printf("Parameter found in environment. ddr_trfc = %d\n", + new_trfc); + if (new_trfc < 160000 || new_trfc > 260000) { + // back to default if out of range + new_trfc = 260000; + } + } + debug("N%d.LMC%d: Adjusting tRFC from %d to %d, for CN78XX Pass 2.x\n", + node, if_num, trfc, new_trfc); + trfc = new_trfc; + } + + twtr = spd_twtr * mtb_psec; + trtp = spd_trtp * mtb_psec; + tfaw = spd_tfaw * mtb_psec; + + debug("Medium Timebase (MTB) : %6d ps\n", + mtb_psec); + debug("Minimum Cycle Time (tckmin) : %6d ps (%ld MT/s)\n", + tckmin, pretty_psecs_to_mts(tckmin)); + debug("Minimum CAS Latency Time (taamin) : %6d ps\n", + taamin); + debug("Write Recovery Time (tWR) : %6d ps\n", + twr); + debug("Minimum RAS to CAS delay (tRCD) : %6d ps\n", + trcd); + debug("Minimum Row Active to Row Active delay (tRRD) : %6d ps\n", + trrd); + debug("Minimum Row Precharge Delay (tRP) : %6d ps\n", + trp); + debug("Minimum Active to Precharge (tRAS) : %6d ps\n", + tras); + debug("Minimum Active to Active/Refresh Delay (tRC) : %6d ps\n", + trc); + debug("Minimum Refresh Recovery Delay (tRFC) : %6d ps\n", + trfc); + debug("Internal write to read command delay (tWTR) : %6d ps\n", + twtr); + debug("Min Internal Rd to Precharge Cmd Delay (tRTP) : %6d ps\n", + trtp); + debug("Minimum Four Activate Window Delay (tFAW) : %6d ps\n", + tfaw); + } + + /* + * When the cycle time is within 1 psec of the minimum accept it + * as a slight rounding error and adjust it to exactly the minimum + * cycle time. This avoids an unnecessary warning. + */ + if (abs(tclk_psecs - tckmin) < 2) + tclk_psecs = tckmin; + + if (tclk_psecs < (u64)tckmin) { + printf("WARNING!!!!: DDR Clock Rate (tCLK: %ld) exceeds DIMM specifications (tckmin: %ld)!!!!\n", + tclk_psecs, (ulong)tckmin); + } + + debug("DDR Clock Rate (tCLK) : %6ld ps\n", + tclk_psecs); + debug("Core Clock Rate (eCLK) : %6ld ps\n", + eclk_psecs); + + s = env_get("ddr_use_ecc"); + if (s) { + use_ecc = !!simple_strtoul(s, NULL, 0); + printf("Parameter found in environment. ddr_use_ecc = %d\n", + use_ecc); + } + use_ecc = use_ecc && spd_ecc; + + if_bytemask = if_64b ? (use_ecc ? 0x1ff : 0xff) + : (use_ecc ? 0x01f : 0x0f); + + debug("DRAM Interface width: %d bits %s bytemask 0x%03x\n", + if_64b ? 64 : 32, use_ecc ? "+ECC" : "", if_bytemask); + + debug("\n------ Board Custom Configuration Settings ------\n"); + debug("%-45s : %d\n", "MIN_RTT_NOM_IDX ", c_cfg->min_rtt_nom_idx); + debug("%-45s : %d\n", "MAX_RTT_NOM_IDX ", c_cfg->max_rtt_nom_idx); + debug("%-45s : %d\n", "MIN_RODT_CTL ", c_cfg->min_rodt_ctl); + debug("%-45s : %d\n", "MAX_RODT_CTL ", c_cfg->max_rodt_ctl); + debug("%-45s : %d\n", "MIN_CAS_LATENCY ", c_cfg->min_cas_latency); + debug("%-45s : %d\n", "OFFSET_EN ", c_cfg->offset_en); + debug("%-45s : %d\n", "OFFSET_UDIMM ", c_cfg->offset_udimm); + debug("%-45s : %d\n", "OFFSET_RDIMM ", c_cfg->offset_rdimm); + debug("%-45s : %d\n", "DDR_RTT_NOM_AUTO ", c_cfg->ddr_rtt_nom_auto); + debug("%-45s : %d\n", "DDR_RODT_CTL_AUTO ", c_cfg->ddr_rodt_ctl_auto); + if (spd_rdimm) + debug("%-45s : %d\n", "RLEVEL_COMP_OFFSET", + c_cfg->rlevel_comp_offset_rdimm); + else + debug("%-45s : %d\n", "RLEVEL_COMP_OFFSET", + c_cfg->rlevel_comp_offset_udimm); + debug("%-45s : %d\n", "RLEVEL_COMPUTE ", c_cfg->rlevel_compute); + debug("%-45s : %d\n", "DDR2T_UDIMM ", c_cfg->ddr2t_udimm); + debug("%-45s : %d\n", "DDR2T_RDIMM ", c_cfg->ddr2t_rdimm); + debug("%-45s : %d\n", "FPRCH2 ", c_cfg->fprch2); + debug("%-45s : %d\n", "PTUNE_OFFSET ", c_cfg->ptune_offset); + debug("%-45s : %d\n", "NTUNE_OFFSET ", c_cfg->ntune_offset); + debug("-------------------------------------------------\n"); + + cl = divide_roundup(taamin, tclk_psecs); + + debug("Desired CAS Latency : %6d\n", cl); + + min_cas_latency = c_cfg->min_cas_latency; + + s = lookup_env(priv, "ddr_min_cas_latency"); + if (s) + min_cas_latency = simple_strtoul(s, NULL, 0); + + debug("CAS Latencies supported in DIMM :"); + base_cl = (ddr_type == DDR4_DRAM) ? 7 : 4; + for (i = 0; i < 32; ++i) { + if ((spd_cas_latency >> i) & 1) { + debug(" %d", i + base_cl); + max_cas_latency = i + base_cl; + if (min_cas_latency == 0) + min_cas_latency = i + base_cl; + } + } + debug("\n"); + + /* + * Use relaxed timing when running slower than the minimum + * supported speed. Adjust timing to match the smallest supported + * CAS Latency. + */ + if (min_cas_latency > cl) { + ulong adjusted_tclk = taamin / min_cas_latency; + + cl = min_cas_latency; + debug("Slow clock speed. Adjusting timing: tClk = %ld, Adjusted tClk = %ld\n", + tclk_psecs, adjusted_tclk); + tclk_psecs = adjusted_tclk; + } + + s = env_get("ddr_cas_latency"); + if (s) { + override_cas_latency = simple_strtoul(s, NULL, 0); + printf("Parameter found in environment. ddr_cas_latency = %d\n", + override_cas_latency); + } + + /* Make sure that the selected cas latency is legal */ + for (i = (cl - base_cl); i < 32; ++i) { + if ((spd_cas_latency >> i) & 1) { + cl = i + base_cl; + break; + } + } + + if (max_cas_latency < cl) + cl = max_cas_latency; + + if (override_cas_latency != 0) + cl = override_cas_latency; + + debug("CAS Latency : %6d\n", cl); + + if ((cl * tckmin) > 20000) { + debug("(CLactual * tckmin) = %d exceeds 20 ns\n", + (cl * tckmin)); + } + + if (tclk_psecs < (ulong)tckmin) { + printf("WARNING!!!!!!: DDR3 Clock Rate (tCLK: %ld) exceeds DIMM specifications (tckmin:%ld)!!!!!!!!\n", + tclk_psecs, (ulong)tckmin); + } + + if (num_banks != 4 && num_banks != 8 && num_banks != 16) { + printf("Unsupported number of banks %d. Must be 4 or 8.\n", + num_banks); + ++fatal_error; + } + + if (num_ranks != 1 && num_ranks != 2 && num_ranks != 4) { + printf("Unsupported number of ranks: %d\n", num_ranks); + ++fatal_error; + } + + if (octeon_is_cpuid(OCTEON_CN78XX) || + octeon_is_cpuid(OCTEON_CN73XX) || + octeon_is_cpuid(OCTEON_CNF75XX)) { + if (dram_width != 8 && dram_width != 16 && dram_width != 4) { + printf("Unsupported SDRAM Width, %d. Must be 4, 8 or 16.\n", + dram_width); + ++fatal_error; + } + } else if (dram_width != 8 && dram_width != 16) { + printf("Unsupported SDRAM Width, %d. Must be 8 or 16.\n", + dram_width); + ++fatal_error; + } + + /* + ** Bail out here if things are not copasetic. + */ + if (fatal_error) + return (-1); + + /* + * 4.8.4 LMC RESET Initialization + * + * The purpose of this step is to assert/deassert the RESET# pin at the + * DDR3/DDR4 parts. + * + * This LMC RESET step is done for all enabled LMCs. + */ + perform_lmc_reset(priv, node, if_num); + + // Make sure scrambling is disabled during init... + ctrl.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(if_num)); + ctrl.s.scramble_ena = 0; + lmc_wr(priv, CVMX_LMCX_CONTROL(if_num), ctrl.u64); + + lmc_wr(priv, CVMX_LMCX_SCRAMBLE_CFG0(if_num), 0); + lmc_wr(priv, CVMX_LMCX_SCRAMBLE_CFG1(if_num), 0); + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS1_X)) + lmc_wr(priv, CVMX_LMCX_SCRAMBLE_CFG2(if_num), 0); + + odt_idx = min(dimm_count - 1, 3); + + switch (num_ranks) { + case 1: + odt_config = odt_1rank_config; + break; + case 2: + odt_config = odt_2rank_config; + break; + case 4: + odt_config = odt_4rank_config; + break; + default: + odt_config = disable_odt_config; + printf("Unsupported number of ranks: %d\n", num_ranks); + ++fatal_error; + } + + /* + * 4.8.5 Early LMC Initialization + * + * All of DDR PLL, LMC CK, and LMC DRESET initializations must be + * completed prior to starting this LMC initialization sequence. + * + * Perform the following five substeps for early LMC initialization: + * + * 1. Software must ensure there are no pending DRAM transactions. + * + * 2. Write LMC(0)_CONFIG, LMC(0)_CONTROL, LMC(0)_TIMING_PARAMS0, + * LMC(0)_TIMING_PARAMS1, LMC(0)_MODEREG_PARAMS0, + * LMC(0)_MODEREG_PARAMS1, LMC(0)_DUAL_MEMCFG, LMC(0)_NXM, + * LMC(0)_WODT_MASK, LMC(0)_RODT_MASK, LMC(0)_COMP_CTL2, + * LMC(0)_PHY_CTL, LMC(0)_DIMM0/1_PARAMS, and LMC(0)_DIMM_CTL with + * appropriate values. All sections in this chapter can be used to + * derive proper register settings. + */ + + /* LMC(0)_CONFIG */ + lmc_config(priv); + + /* LMC(0)_CONTROL */ + lmc_control(priv); + + /* LMC(0)_TIMING_PARAMS0 */ + lmc_timing_params0(priv); + + /* LMC(0)_TIMING_PARAMS1 */ + lmc_timing_params1(priv); + + /* LMC(0)_TIMING_PARAMS2 */ + lmc_timing_params2(priv); + + /* LMC(0)_MODEREG_PARAMS0 */ + lmc_modereg_params0(priv); + + /* LMC(0)_MODEREG_PARAMS1 */ + lmc_modereg_params1(priv); + + /* LMC(0)_MODEREG_PARAMS2 */ + lmc_modereg_params2(priv); + + /* LMC(0)_MODEREG_PARAMS3 */ + lmc_modereg_params3(priv); + + /* LMC(0)_NXM */ + lmc_nxm(priv); + + /* LMC(0)_WODT_MASK */ + lmc_wodt_mask(priv); + + /* LMC(0)_RODT_MASK */ + lmc_rodt_mask(priv); + + /* LMC(0)_COMP_CTL2 */ + lmc_comp_ctl2(priv); + + /* LMC(0)_PHY_CTL */ + lmc_phy_ctl(priv); + + /* LMC(0)_EXT_CONFIG */ + lmc_ext_config(priv); + + /* LMC(0)_EXT_CONFIG2 */ + lmc_ext_config2(priv); + + /* LMC(0)_DIMM0/1_PARAMS */ + lmc_dimm01_params(priv); + + ret = lmc_rank_init(priv); + if (ret < 0) + return 0; /* 0 indicates problem */ + + lmc_config_2(priv); + + lmc_write_leveling(priv); + + lmc_read_leveling(priv); + + lmc_workaround(priv); + + ret = lmc_sw_write_leveling(priv); + if (ret < 0) + return 0; /* 0 indicates problem */ + + // this sometimes causes stack overflow crashes.. + // display only for DDR4 RDIMMs. + if (ddr_type == DDR4_DRAM && spd_rdimm) { + int i; + + for (i = 0; i < 3; i += 2) // just pages 0 and 2 for now.. + display_mpr_page(priv, rank_mask, if_num, i); + } + + lmc_dll(priv); + + lmc_workaround_2(priv); + + lmc_final(priv); + + lmc_scrambling(priv); + + return mem_size_mbytes; +} + +///// HW-assist byte DLL offset tuning ////// + +static int cvmx_dram_get_num_lmc(struct ddr_priv *priv) +{ + union cvmx_lmcx_dll_ctl2 lmcx_dll_ctl2; + + if (octeon_is_cpuid(OCTEON_CN70XX)) + return 1; + + if (octeon_is_cpuid(OCTEON_CN73XX) || octeon_is_cpuid(OCTEON_CNF75XX)) { + // sample LMC1 + lmcx_dll_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL2(1)); + if (lmcx_dll_ctl2.cn78xx.intf_en) + return 2; + else + return 1; + } + + // for CN78XX, LMCs are always active in pairs, and always LMC0/1 + // so, we sample LMC2 to see if 2 and 3 are active + lmcx_dll_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL2(2)); + if (lmcx_dll_ctl2.cn78xx.intf_en) + return 4; + else + return 2; +} + +// got to do these here, even though already defined in BDK + +// all DDR3, and DDR4 x16 today, use only 3 bank bits; +// DDR4 x4 and x8 always have 4 bank bits +// NOTE: this will change in the future, when DDR4 x16 devices can +// come with 16 banks!! FIXME!! +static int cvmx_dram_get_num_bank_bits(struct ddr_priv *priv, int lmc) +{ + union cvmx_lmcx_dll_ctl2 lmcx_dll_ctl2; + union cvmx_lmcx_config lmcx_config; + union cvmx_lmcx_ddr_pll_ctl lmcx_ddr_pll_ctl; + int bank_width; + + // can always read this + lmcx_dll_ctl2.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL2(lmc)); + + if (lmcx_dll_ctl2.cn78xx.dreset) // check LMCn + return 0; + + lmcx_config.u64 = lmc_rd(priv, CVMX_LMCX_DLL_CTL2(lmc)); + lmcx_ddr_pll_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DDR_PLL_CTL(lmc)); + + bank_width = ((lmcx_ddr_pll_ctl.s.ddr4_mode != 0) && + (lmcx_config.s.bg2_enable)) ? 4 : 3; + + return bank_width; +} + +#define EXTRACT(v, lsb, width) (((v) >> (lsb)) & ((1ull << (width)) - 1)) +#define ADDRESS_HOLE 0x10000000ULL + +static void cvmx_dram_address_extract_info(struct ddr_priv *priv, u64 address, + int *node, int *lmc, int *dimm, + int *prank, int *lrank, int *bank, + int *row, int *col) +{ + int bank_lsb, xbits; + union cvmx_l2c_ctl l2c_ctl; + union cvmx_lmcx_config lmcx_config; + union cvmx_lmcx_control lmcx_control; + union cvmx_lmcx_ext_config ext_config; + int bitno = (octeon_is_cpuid(OCTEON_CN7XXX)) ? 20 : 18; + int bank_width; + int dimm_lsb; + int dimm_width; + int prank_lsb, lrank_lsb; + int prank_width, lrank_width; + int row_lsb; + int row_width; + int col_hi_lsb; + int col_hi_width; + int col_hi; + + if (octeon_is_cpuid(OCTEON_CN73XX) || octeon_is_cpuid(OCTEON_CNF75XX)) + bitno = 18; + + *node = EXTRACT(address, 40, 2); /* Address bits [41:40] */ + + address &= (1ULL << 40) - 1; // lop off any node bits or above + if (address >= ADDRESS_HOLE) // adjust down if at HOLE or above + address -= ADDRESS_HOLE; + + /* Determine the LMC controllers */ + l2c_ctl.u64 = l2c_rd(priv, CVMX_L2C_CTL); + + /* xbits depends on number of LMCs */ + xbits = cvmx_dram_get_num_lmc(priv) >> 1; // 4->2, 2->1, 1->0 + bank_lsb = 7 + xbits; + + /* LMC number is probably aliased */ + if (l2c_ctl.s.disidxalias) { + *lmc = EXTRACT(address, 7, xbits); + } else { + *lmc = EXTRACT(address, 7, xbits) ^ + EXTRACT(address, bitno, xbits) ^ + EXTRACT(address, 12, xbits); + } + + /* Figure out the bank field width */ + lmcx_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(*lmc)); + ext_config.u64 = lmc_rd(priv, CVMX_LMCX_EXT_CONFIG(*lmc)); + bank_width = cvmx_dram_get_num_bank_bits(priv, *lmc); + + /* Extract additional info from the LMC_CONFIG CSR */ + dimm_lsb = 28 + lmcx_config.s.pbank_lsb + xbits; + dimm_width = 40 - dimm_lsb; + prank_lsb = dimm_lsb - lmcx_config.s.rank_ena; + prank_width = dimm_lsb - prank_lsb; + lrank_lsb = prank_lsb - ext_config.s.dimm0_cid; + lrank_width = prank_lsb - lrank_lsb; + row_lsb = 14 + lmcx_config.s.row_lsb + xbits; + row_width = lrank_lsb - row_lsb; + col_hi_lsb = bank_lsb + bank_width; + col_hi_width = row_lsb - col_hi_lsb; + + /* Extract the parts of the address */ + *dimm = EXTRACT(address, dimm_lsb, dimm_width); + *prank = EXTRACT(address, prank_lsb, prank_width); + *lrank = EXTRACT(address, lrank_lsb, lrank_width); + *row = EXTRACT(address, row_lsb, row_width); + + /* bank calculation may be aliased... */ + lmcx_control.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(*lmc)); + if (lmcx_control.s.xor_bank) { + *bank = EXTRACT(address, bank_lsb, bank_width) ^ + EXTRACT(address, 12 + xbits, bank_width); + } else { + *bank = EXTRACT(address, bank_lsb, bank_width); + } + + /* LMC number already extracted */ + col_hi = EXTRACT(address, col_hi_lsb, col_hi_width); + *col = EXTRACT(address, 3, 4) | (col_hi << 4); + /* Bus byte is address bits [2:0]. Unused here */ +} + +// end of added workarounds + +// NOTE: "mode" argument: +// DBTRAIN_TEST: for testing using GP patterns, includes ECC +// DBTRAIN_DBI: for DBI deskew training behavior (uses GP patterns) +// DBTRAIN_LFSR: for testing using LFSR patterns, includes ECC +// NOTE: trust the caller to specify the correct/supported mode +// +static int test_dram_byte_hw(struct ddr_priv *priv, int if_num, u64 p, + int mode, u64 *xor_data) +{ + u64 p1; + u64 k; + int errors = 0; + + u64 mpr_data0, mpr_data1; + u64 bad_bits[2] = { 0, 0 }; + + int node_address, lmc, dimm; + int prank, lrank; + int bank, row, col; + int save_or_dis; + int byte; + int ba_loop, ba_bits; + + union cvmx_lmcx_rlevel_ctl rlevel_ctl; + union cvmx_lmcx_dbtrain_ctl dbtrain_ctl; + union cvmx_lmcx_phy_ctl phy_ctl; + + int biter_errs; + + // FIXME: K iterations set to 4 for now. + // FIXME: decrement to increase interations. + // FIXME: must be no less than 22 to stay above an LMC hash field. + int kshift = 27; + + const char *s; + int node = 0; + + // allow override default setting for kshift + s = env_get("ddr_tune_set_kshift"); + if (s) { + int temp = simple_strtoul(s, NULL, 0); + + if (temp < 22 || temp > 28) { + debug("N%d.LMC%d: ILLEGAL override of kshift to %d, using default %d\n", + node, if_num, temp, kshift); + } else { + debug("N%d.LMC%d: overriding kshift (%d) to %d\n", + node, if_num, kshift, temp); + kshift = temp; + } + } + + /* + * 1) Make sure that RLEVEL_CTL[OR_DIS] = 0. + */ + rlevel_ctl.u64 = lmc_rd(priv, CVMX_LMCX_RLEVEL_CTL(if_num)); + save_or_dis = rlevel_ctl.s.or_dis; + /* or_dis must be disabled for this sequence */ + rlevel_ctl.s.or_dis = 0; + lmc_wr(priv, CVMX_LMCX_RLEVEL_CTL(if_num), rlevel_ctl.u64); + + /* + * NOTE: this step done in the calling routine(s)... + * 3) Setup GENERAL_PURPOSE[0-2] registers with the data pattern + * of choice. + * a. GENERAL_PURPOSE0[DATA<63:0>] – sets the initial lower + * (rising edge) 64 bits of data. + * b. GENERAL_PURPOSE1[DATA<63:0>] – sets the initial upper + * (falling edge) 64 bits of data. + * c. GENERAL_PURPOSE2[DATA<15:0>] – sets the initial lower + * (rising edge <7:0>) and upper (falling edge <15:8>) ECC data. + */ + + // final address must include LMC and node + p |= (if_num << 7); /* Map address into proper interface */ + p |= (u64)node << CVMX_NODE_MEM_SHIFT; // map to node + + /* + * Add base offset to both test regions to not clobber u-boot stuff + * when running from L2 for NAND boot. + */ + p += 0x20000000; // offset to 512MB, ie above THE HOLE!!! + p |= 1ull << 63; // needed for OCTEON + + errors = 0; + + cvmx_dram_address_extract_info(priv, p, &node_address, &lmc, &dimm, + &prank, &lrank, &bank, &row, &col); + debug("%s: START at A:0x%012llx, N%d L%d D%d/%d R%d B%1x Row:%05x Col:%05x\n", + __func__, p, node_address, lmc, dimm, prank, lrank, bank, + row, col); + + // only check once per call, and ignore if no match... + if ((int)node != node_address) { + printf("ERROR: Node address mismatch\n"); + return 0; + } + if (lmc != if_num) { + printf("ERROR: LMC address mismatch\n"); + return 0; + } + + /* + * 7) Set PHY_CTL[PHY_RESET] = 1 (LMC automatically clears this as + * it’s a one-shot operation). This is to get into the habit of + * resetting PHY’s SILO to the original 0 location. + */ + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.phy_reset = 1; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + /* + * Walk through a range of addresses avoiding bits that alias + * interfaces on the CN88XX. + */ + + // FIXME: want to try to keep the K increment from affecting the + // LMC via hash, so keep it above bit 21 we also want to keep k + // less than the base offset of bit 29 (512MB) + + for (k = 0; k < (1UL << 29); k += (1UL << kshift)) { + // FIXME: the sequence will interate over 1/2 cacheline + // FIXME: for each unit specified in "read_cmd_count", + // FIXME: so, we setup each sequence to do the max cachelines + // it can + + p1 = p + k; + + cvmx_dram_address_extract_info(priv, p1, &node_address, &lmc, + &dimm, &prank, &lrank, &bank, + &row, &col); + + /* + * 2) Setup the fields of the CSR DBTRAIN_CTL as follows: + * a. COL, ROW, BA, BG, PRANK points to the starting point + * of the address. + * You can just set them to all 0. + * b. RW_TRAIN – set this to 1. + * c. TCCD_L – set this to 0. + * d. READ_CMD_COUNT – instruct the sequence to the how many + * writes/reads. + * It is 5 bits field, so set to 31 of maximum # of r/w. + */ + dbtrain_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DBTRAIN_CTL(if_num)); + dbtrain_ctl.s.column_a = col; + dbtrain_ctl.s.row_a = row; + dbtrain_ctl.s.bg = (bank >> 2) & 3; + dbtrain_ctl.s.prank = (dimm * 2) + prank; // FIXME? + dbtrain_ctl.s.lrank = lrank; // FIXME? + dbtrain_ctl.s.activate = (mode == DBTRAIN_DBI); + dbtrain_ctl.s.write_ena = 1; + dbtrain_ctl.s.read_cmd_count = 31; // max count pass 1.x + if (octeon_is_cpuid(OCTEON_CN78XX_PASS2_X) || + octeon_is_cpuid(OCTEON_CNF75XX)) { + // max count on chips that support it + dbtrain_ctl.s.cmd_count_ext = 3; + } else { + // max count pass 1.x + dbtrain_ctl.s.cmd_count_ext = 0; + } + + dbtrain_ctl.s.rw_train = 1; + dbtrain_ctl.s.tccd_sel = (mode == DBTRAIN_DBI); + // LFSR should only be on when chip supports it... + dbtrain_ctl.s.lfsr_pattern_sel = (mode == DBTRAIN_LFSR) ? 1 : 0; + + biter_errs = 0; + + // for each address, iterate over the 4 "banks" in the BA + for (ba_loop = 0, ba_bits = bank & 3; + ba_loop < 4; ba_loop++, ba_bits = (ba_bits + 1) & 3) { + dbtrain_ctl.s.ba = ba_bits; + lmc_wr(priv, CVMX_LMCX_DBTRAIN_CTL(if_num), + dbtrain_ctl.u64); + + /* + * We will use the RW_TRAINING sequence (14) for + * this task. + * + * 4) Kick off the sequence (SEQ_CTL[SEQ_SEL] = 14, + * SEQ_CTL[INIT_START] = 1). + * 5) Poll on SEQ_CTL[SEQ_COMPLETE] for completion. + */ + oct3_ddr3_seq(priv, prank, if_num, 14); + + /* + * 6) Read MPR_DATA0 and MPR_DATA1 for results. + * a. MPR_DATA0[MPR_DATA<63:0>] – comparison results + * for DQ63:DQ0. (1 means MATCH, 0 means FAIL). + * b. MPR_DATA1[MPR_DATA<7:0>] – comparison results + * for ECC bit7:0. + */ + mpr_data0 = lmc_rd(priv, CVMX_LMCX_MPR_DATA0(if_num)); + mpr_data1 = lmc_rd(priv, CVMX_LMCX_MPR_DATA1(if_num)); + + /* + * 7) Set PHY_CTL[PHY_RESET] = 1 (LMC automatically + * clears this as it’s a one-shot operation). + * This is to get into the habit of resetting PHY’s + * SILO to the original 0 location. + */ + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(if_num)); + phy_ctl.s.phy_reset = 1; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(if_num), phy_ctl.u64); + + // bypass any error checking or updating when DBI mode + if (mode == DBTRAIN_DBI) + continue; + + // data bytes + if (~mpr_data0) { + for (byte = 0; byte < 8; byte++) { + if ((~mpr_data0 >> (8 * byte)) & 0xffUL) + biter_errs |= (1 << byte); + } + // accumulate bad bits + bad_bits[0] |= ~mpr_data0; + } + + // include ECC byte errors + if (~mpr_data1 & 0xffUL) { + biter_errs |= (1 << 8); + bad_bits[1] |= ~mpr_data1 & 0xffUL; + } + } + + errors |= biter_errs; + } /* end for (k=...) */ + + rlevel_ctl.s.or_dis = save_or_dis; + lmc_wr(priv, CVMX_LMCX_RLEVEL_CTL(if_num), rlevel_ctl.u64); + + // send the bad bits back... + if (mode != DBTRAIN_DBI && xor_data) { + xor_data[0] = bad_bits[0]; + xor_data[1] = bad_bits[1]; + } + + return errors; +} + +// setup default for byte test pattern array +// take these from the HRM section 6.9.13 +static const u64 byte_pattern_0[] = { + 0xFFAAFFFFFF55FFFFULL, // GP0 + 0x55555555AAAAAAAAULL, // GP1 + 0xAA55AAAAULL, // GP2 +}; + +static const u64 byte_pattern_1[] = { + 0xFBF7EFDFBF7FFEFDULL, // GP0 + 0x0F1E3C78F0E1C387ULL, // GP1 + 0xF0E1BF7FULL, // GP2 +}; + +// this is from Andrew via LFSR with PRBS=0xFFFFAAAA +static const u64 byte_pattern_2[] = { + 0xEE55AADDEE55AADDULL, // GP0 + 0x55AADDEE55AADDEEULL, // GP1 + 0x55EEULL, // GP2 +}; + +// this is from Mike via LFSR with PRBS=0x4A519909 +static const u64 byte_pattern_3[] = { + 0x0088CCEE0088CCEEULL, // GP0 + 0xBB552211BB552211ULL, // GP1 + 0xBB00ULL, // GP2 +}; + +static const u64 *byte_patterns[4] = { + byte_pattern_0, byte_pattern_1, byte_pattern_2, byte_pattern_3 +}; + +static const u32 lfsr_patterns[4] = { + 0xFFFFAAAAUL, 0x06000000UL, 0xAAAAFFFFUL, 0x4A519909UL +}; + +#define NUM_BYTE_PATTERNS 4 + +#define DEFAULT_BYTE_BURSTS 32 // compromise between time and rigor + +static void setup_hw_pattern(struct ddr_priv *priv, int lmc, + const u64 *pattern_p) +{ + /* + * 3) Setup GENERAL_PURPOSE[0-2] registers with the data pattern + * of choice. + * a. GENERAL_PURPOSE0[DATA<63:0>] – sets the initial lower + * (rising edge) 64 bits of data. + * b. GENERAL_PURPOSE1[DATA<63:0>] – sets the initial upper + * (falling edge) 64 bits of data. + * c. GENERAL_PURPOSE2[DATA<15:0>] – sets the initial lower + * (rising edge <7:0>) and upper + * (falling edge <15:8>) ECC data. + */ + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE0(lmc), pattern_p[0]); + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE1(lmc), pattern_p[1]); + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE2(lmc), pattern_p[2]); +} + +static void setup_lfsr_pattern(struct ddr_priv *priv, int lmc, u32 data) +{ + union cvmx_lmcx_char_ctl char_ctl; + u32 prbs; + const char *s; + + s = env_get("ddr_lfsr_prbs"); + if (s) + prbs = simple_strtoul(s, NULL, 0); + else + prbs = data; + + /* + * 2) DBTRAIN_CTL[LFSR_PATTERN_SEL] = 1 + * here data comes from the LFSR generating a PRBS pattern + * CHAR_CTL.EN = 0 + * CHAR_CTL.SEL = 0; // for PRBS + * CHAR_CTL.DR = 1; + * CHAR_CTL.PRBS = setup for whatever type of PRBS to send + * CHAR_CTL.SKEW_ON = 1; + */ + char_ctl.u64 = lmc_rd(priv, CVMX_LMCX_CHAR_CTL(lmc)); + char_ctl.s.en = 0; + char_ctl.s.sel = 0; + char_ctl.s.dr = 1; + char_ctl.s.prbs = prbs; + char_ctl.s.skew_on = 1; + lmc_wr(priv, CVMX_LMCX_CHAR_CTL(lmc), char_ctl.u64); +} + +static int choose_best_hw_patterns(int lmc, int mode) +{ + int new_mode = mode; + const char *s; + + switch (mode) { + case DBTRAIN_TEST: // always choose LFSR if chip supports it + if (octeon_is_cpuid(OCTEON_CN78XX_PASS2_X)) { + int lfsr_enable = 1; + + s = env_get("ddr_allow_lfsr"); + if (s) { + // override? + lfsr_enable = !!strtoul(s, NULL, 0); + } + + if (lfsr_enable) + new_mode = DBTRAIN_LFSR; + } + break; + + case DBTRAIN_DBI: // possibly can allow LFSR use? + break; + + case DBTRAIN_LFSR: // forced already + if (!octeon_is_cpuid(OCTEON_CN78XX_PASS2_X)) { + debug("ERROR: illegal HW assist mode %d\n", mode); + new_mode = DBTRAIN_TEST; + } + break; + + default: + debug("ERROR: unknown HW assist mode %d\n", mode); + } + + if (new_mode != mode) + debug("%s: changing mode %d to %d\n", __func__, mode, new_mode); + + return new_mode; +} + +int run_best_hw_patterns(struct ddr_priv *priv, int lmc, u64 phys_addr, + int mode, u64 *xor_data) +{ + int pattern; + const u64 *pattern_p; + int errs, errors = 0; + + // FIXME? always choose LFSR if chip supports it??? + mode = choose_best_hw_patterns(lmc, mode); + + for (pattern = 0; pattern < NUM_BYTE_PATTERNS; pattern++) { + if (mode == DBTRAIN_LFSR) { + setup_lfsr_pattern(priv, lmc, lfsr_patterns[pattern]); + } else { + pattern_p = byte_patterns[pattern]; + setup_hw_pattern(priv, lmc, pattern_p); + } + errs = test_dram_byte_hw(priv, lmc, phys_addr, mode, xor_data); + + debug("%s: PATTERN %d at A:0x%012llx errors 0x%x\n", + __func__, pattern, phys_addr, errs); + + errors |= errs; + } + + return errors; +} + +static void hw_assist_test_dll_offset(struct ddr_priv *priv, + int dll_offset_mode, int lmc, + int bytelane, + int if_64b, + u64 dram_tune_rank_offset, + int dram_tune_byte_bursts) +{ + int byte_offset, new_best_offset[9]; + int rank_delay_start[4][9]; + int rank_delay_count[4][9]; + int rank_delay_best_start[4][9]; + int rank_delay_best_count[4][9]; + int errors[4], off_errors, tot_errors; + int rank_mask, rankx, active_ranks; + int pattern; + const u64 *pattern_p; + int byte; + char *mode_str = (dll_offset_mode == 2) ? "Read" : "Write"; + int pat_best_offset[9]; + u64 phys_addr; + int pat_beg, pat_end; + int rank_beg, rank_end; + int byte_lo, byte_hi; + union cvmx_lmcx_config lmcx_config; + u64 hw_rank_offset; + int num_lmcs = cvmx_dram_get_num_lmc(priv); + // FIXME? always choose LFSR if chip supports it??? + int mode = choose_best_hw_patterns(lmc, DBTRAIN_TEST); + int node = 0; + + if (bytelane == 0x0A) { // all bytelanes + byte_lo = 0; + byte_hi = 8; + } else { // just 1 + byte_lo = bytelane; + byte_hi = bytelane; + } + + lmcx_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(lmc)); + rank_mask = lmcx_config.s.init_status; + + // this should be correct for 1 or 2 ranks, 1 or 2 DIMMs + hw_rank_offset = + 1ull << (28 + lmcx_config.s.pbank_lsb - lmcx_config.s.rank_ena + + (num_lmcs / 2)); + + debug("N%d: %s: starting LMC%d with rank offset 0x%016llx\n", + node, __func__, lmc, (unsigned long long)hw_rank_offset); + + // start of pattern loop + // we do the set of tests for each pattern supplied... + + memset(new_best_offset, 0, sizeof(new_best_offset)); + for (pattern = 0; pattern < NUM_BYTE_PATTERNS; pattern++) { + memset(pat_best_offset, 0, sizeof(pat_best_offset)); + + if (mode == DBTRAIN_TEST) { + pattern_p = byte_patterns[pattern]; + setup_hw_pattern(priv, lmc, pattern_p); + } else { + setup_lfsr_pattern(priv, lmc, lfsr_patterns[pattern]); + } + + // now loop through all legal values for the DLL byte offset... + +#define BYTE_OFFSET_INCR 3 // FIXME: make this tunable? + + tot_errors = 0; + + memset(rank_delay_count, 0, sizeof(rank_delay_count)); + memset(rank_delay_start, 0, sizeof(rank_delay_start)); + memset(rank_delay_best_count, 0, sizeof(rank_delay_best_count)); + memset(rank_delay_best_start, 0, sizeof(rank_delay_best_start)); + + for (byte_offset = -63; byte_offset < 64; + byte_offset += BYTE_OFFSET_INCR) { + // do the setup on the active LMC + // set the bytelanes DLL offsets + change_dll_offset_enable(priv, lmc, 0); + // FIXME? bytelane? + load_dll_offset(priv, lmc, dll_offset_mode, + byte_offset, bytelane); + change_dll_offset_enable(priv, lmc, 1); + + //bdk_watchdog_poke(); + + // run the test on each rank + // only 1 call per rank should be enough, let the + // bursts, loops, etc, control the load... + + // errors for this byte_offset, all ranks + off_errors = 0; + + active_ranks = 0; + + for (rankx = 0; rankx < 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + phys_addr = hw_rank_offset * active_ranks; + // FIXME: now done by test_dram_byte_hw() + //phys_addr |= (lmc << 7); + //phys_addr |= (u64)node << CVMX_NODE_MEM_SHIFT; + + active_ranks++; + + // NOTE: return is a now a bitmask of the + // erroring bytelanes. + errors[rankx] = + test_dram_byte_hw(priv, lmc, phys_addr, + mode, NULL); + + // process any errors in the bytelane(s) that + // are being tested + for (byte = byte_lo; byte <= byte_hi; byte++) { + // check errors + // yes, an error in the byte lane in + // this rank + if (errors[rankx] & (1 << byte)) { + off_errors |= (1 << byte); + + debug("N%d.LMC%d.R%d: Bytelane %d DLL %s Offset Test %3d: Address 0x%012llx errors\n", + node, lmc, rankx, byte, + mode_str, byte_offset, + phys_addr); + + // had started run + if (rank_delay_count + [rankx][byte] > 0) { + debug("N%d.LMC%d.R%d: Bytelane %d DLL %s Offset Test %3d: stopping a run here\n", + node, lmc, rankx, + byte, mode_str, + byte_offset); + // stop now + rank_delay_count + [rankx][byte] = + 0; + } + // FIXME: else had not started + // run - nothing else to do? + } else { + // no error in the byte lane + // first success, set run start + if (rank_delay_count[rankx] + [byte] == 0) { + debug("N%d.LMC%d.R%d: Bytelane %d DLL %s Offset Test %3d: starting a run here\n", + node, lmc, rankx, + byte, mode_str, + byte_offset); + rank_delay_start[rankx] + [byte] = + byte_offset; + } + // bump run length + rank_delay_count[rankx][byte] + += BYTE_OFFSET_INCR; + + // is this now the biggest + // window? + if (rank_delay_count[rankx] + [byte] > + rank_delay_best_count[rankx] + [byte]) { + rank_delay_best_count + [rankx][byte] = + rank_delay_count + [rankx][byte]; + rank_delay_best_start + [rankx][byte] = + rank_delay_start + [rankx][byte]; + debug("N%d.LMC%d.R%d: Bytelane %d DLL %s Offset Test %3d: updating best to %d/%d\n", + node, lmc, rankx, + byte, mode_str, + byte_offset, + rank_delay_best_start + [rankx][byte], + rank_delay_best_count + [rankx][byte]); + } + } + } + } /* for (rankx = 0; rankx < 4; rankx++) */ + + tot_errors |= off_errors; + } + + // set the bytelanes DLL offsets all back to 0 + change_dll_offset_enable(priv, lmc, 0); + load_dll_offset(priv, lmc, dll_offset_mode, 0, bytelane); + change_dll_offset_enable(priv, lmc, 1); + + // now choose the best byte_offsets for this pattern + // according to the best windows of the tested ranks + // calculate offset by constructing an average window + // from the rank windows + for (byte = byte_lo; byte <= byte_hi; byte++) { + pat_beg = -999; + pat_end = 999; + + for (rankx = 0; rankx < 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + rank_beg = rank_delay_best_start[rankx][byte]; + pat_beg = max(pat_beg, rank_beg); + rank_end = rank_beg + + rank_delay_best_count[rankx][byte] - + BYTE_OFFSET_INCR; + pat_end = min(pat_end, rank_end); + + debug("N%d.LMC%d.R%d: Bytelane %d DLL %s Offset Test: Rank Window %3d:%3d\n", + node, lmc, rankx, byte, mode_str, + rank_beg, rank_end); + + } /* for (rankx = 0; rankx < 4; rankx++) */ + + pat_best_offset[byte] = (pat_end + pat_beg) / 2; + + // sum the pattern averages + new_best_offset[byte] += pat_best_offset[byte]; + } + + // now print them on 1 line, descending order... + debug("N%d.LMC%d: HW DLL %s Offset Pattern %d :", + node, lmc, mode_str, pattern); + for (byte = byte_hi; byte >= byte_lo; --byte) + debug(" %4d", pat_best_offset[byte]); + debug("\n"); + } + // end of pattern loop + + debug("N%d.LMC%d: HW DLL %s Offset Average : ", node, lmc, mode_str); + + // print in decending byte index order + for (byte = byte_hi; byte >= byte_lo; --byte) { + // create the new average NINT + new_best_offset[byte] = divide_nint(new_best_offset[byte], + NUM_BYTE_PATTERNS); + + // print the best offsets from all patterns + + // print just the offset of all the bytes + if (bytelane == 0x0A) + debug("%4d ", new_best_offset[byte]); + else // print the bytelanes also + debug("(byte %d) %4d ", byte, new_best_offset[byte]); + + // done with testing, load up the best offsets we found... + // disable offsets while we load... + change_dll_offset_enable(priv, lmc, 0); + load_dll_offset(priv, lmc, dll_offset_mode, + new_best_offset[byte], byte); + // re-enable the offsets now that we are done loading + change_dll_offset_enable(priv, lmc, 1); + } + + debug("\n"); +} + +/* + * Automatically adjust the DLL offset for the selected bytelane using + * hardware-assist + */ +static int perform_HW_dll_offset_tuning(struct ddr_priv *priv, + int dll_offset_mode, int bytelane) +{ + int if_64b; + int save_ecc_ena[4]; + union cvmx_lmcx_config lmc_config; + int lmc, num_lmcs = cvmx_dram_get_num_lmc(priv); + const char *s; + int loops = 1, loop; + int by; + u64 dram_tune_rank_offset; + int dram_tune_byte_bursts = DEFAULT_BYTE_BURSTS; + int node = 0; + + // see if we want to do the tuning more than once per LMC... + s = env_get("ddr_tune_ecc_loops"); + if (s) + loops = strtoul(s, NULL, 0); + + // allow override of the test repeats (bursts) + s = env_get("ddr_tune_byte_bursts"); + if (s) + dram_tune_byte_bursts = strtoul(s, NULL, 10); + + // print current working values + debug("N%d: H/W Tuning for bytelane %d will use %d loops, %d bursts, and %d patterns.\n", + node, bytelane, loops, dram_tune_byte_bursts, NUM_BYTE_PATTERNS); + + // FIXME? get flag from LMC0 only + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(0)); + if_64b = !lmc_config.s.mode32b; + + // this should be correct for 1 or 2 ranks, 1 or 2 DIMMs + dram_tune_rank_offset = + 1ull << (28 + lmc_config.s.pbank_lsb - lmc_config.s.rank_ena + + (num_lmcs / 2)); + + // do once for each active LMC + + for (lmc = 0; lmc < num_lmcs; lmc++) { + debug("N%d: H/W Tuning: starting LMC%d bytelane %d tune.\n", + node, lmc, bytelane); + + /* Enable ECC for the HW tests */ + // NOTE: we do enable ECC, but the HW tests used will not + // generate "visible" errors + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(lmc)); + save_ecc_ena[lmc] = lmc_config.s.ecc_ena; + lmc_config.s.ecc_ena = 1; + lmc_wr(priv, CVMX_LMCX_CONFIG(lmc), lmc_config.u64); + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(lmc)); + + // testing is done on a single LMC at a time + // FIXME: for now, loop here to show what happens multiple times + for (loop = 0; loop < loops; loop++) { + /* Perform DLL offset tuning */ + hw_assist_test_dll_offset(priv, 2 /* 2=read */, lmc, + bytelane, + if_64b, dram_tune_rank_offset, + dram_tune_byte_bursts); + } + + // perform cleanup on active LMC + debug("N%d: H/W Tuning: finishing LMC%d bytelane %d tune.\n", + node, lmc, bytelane); + + /* Restore ECC for DRAM tests */ + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(lmc)); + lmc_config.s.ecc_ena = save_ecc_ena[lmc]; + lmc_wr(priv, CVMX_LMCX_CONFIG(lmc), lmc_config.u64); + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(lmc)); + + // finally, see if there are any read offset overrides + // after tuning + for (by = 0; by < 9; by++) { + s = lookup_env(priv, "ddr%d_tune_byte%d", lmc, by); + if (s) { + int dllro = strtoul(s, NULL, 10); + + change_dll_offset_enable(priv, lmc, 0); + load_dll_offset(priv, lmc, 2, dllro, by); + change_dll_offset_enable(priv, lmc, 1); + } + } + + } /* for (lmc = 0; lmc < num_lmcs; lmc++) */ + + // finish up... + + return 0; + +} /* perform_HW_dll_offset_tuning */ + +// this routine simply makes the calls to the tuning routine and returns +// any errors +static int cvmx_tune_node(struct ddr_priv *priv) +{ + int errs, tot_errs; + int do_dllwo = 0; // default to NO + const char *str; + int node = 0; + + // Automatically tune the data and ECC byte DLL read offsets + debug("N%d: Starting DLL Read Offset Tuning for LMCs\n", node); + errs = perform_HW_dll_offset_tuning(priv, 2, 0x0A /* all bytelanes */); + debug("N%d: Finished DLL Read Offset Tuning for LMCs, %d errors\n", + node, errs); + tot_errs = errs; + + // disabled by default for now, does not seem to be needed? + // Automatically tune the data and ECC byte DLL write offsets + // allow override of default setting + str = env_get("ddr_tune_write_offsets"); + if (str) + do_dllwo = !!strtoul(str, NULL, 0); + if (do_dllwo) { + debug("N%d: Starting DLL Write Offset Tuning for LMCs\n", node); + errs = + perform_HW_dll_offset_tuning(priv, 1, + 0x0A /* all bytelanes */); + debug("N%d: Finished DLL Write Offset Tuning for LMCs, %d errors\n", + node, errs); + tot_errs += errs; + } + + return tot_errs; +} + +// this routine makes the calls to the tuning routines when criteria are met +// intended to be called for automated tuning, to apply filtering... + +#define IS_DDR4 1 +#define IS_DDR3 0 +#define IS_RDIMM 1 +#define IS_UDIMM 0 +#define IS_1SLOT 1 +#define IS_2SLOT 0 + +// FIXME: DDR3 is not tuned +static const u32 ddr_speed_filter[2][2][2] = { + [IS_DDR4] = { + [IS_RDIMM] = { + [IS_1SLOT] = 940, + [IS_2SLOT] = 800}, + [IS_UDIMM] = { + [IS_1SLOT] = 1050, + [IS_2SLOT] = 940}, + }, + [IS_DDR3] = { + [IS_RDIMM] = { + [IS_1SLOT] = 0, // disabled + [IS_2SLOT] = 0 // disabled + }, + [IS_UDIMM] = { + [IS_1SLOT] = 0, // disabled + [IS_2SLOT] = 0 // disabled + } + } +}; + +void cvmx_maybe_tune_node(struct ddr_priv *priv, u32 ddr_speed) +{ + const char *s; + union cvmx_lmcx_config lmc_config; + union cvmx_lmcx_control lmc_control; + union cvmx_lmcx_ddr_pll_ctl lmc_ddr_pll_ctl; + int is_ddr4; + int is_rdimm; + int is_1slot; + int do_tune = 0; + u32 ddr_min_speed; + int node = 0; + + // scale it down from Hz to MHz + ddr_speed = divide_nint(ddr_speed, 1000000); + + // FIXME: allow an override here so that all configs can be tuned + // or none + // If the envvar is defined, always either force it or avoid it + // accordingly + s = env_get("ddr_tune_all_configs"); + if (s) { + do_tune = !!strtoul(s, NULL, 0); + printf("N%d: DRAM auto-tuning %s.\n", node, + (do_tune) ? "forced" : "disabled"); + if (do_tune) + cvmx_tune_node(priv); + + return; + } + + // filter the tuning calls here... + // determine if we should/can run automatically for this configuration + // + // FIXME: tune only when the configuration indicates it will help: + // DDR type, RDIMM or UDIMM, 1-slot or 2-slot, and speed + // + lmc_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(0)); // sample LMC0 + lmc_control.u64 = lmc_rd(priv, CVMX_LMCX_CONTROL(0)); // sample LMC0 + // sample LMC0 + lmc_ddr_pll_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DDR_PLL_CTL(0)); + + is_ddr4 = (lmc_ddr_pll_ctl.s.ddr4_mode != 0); + is_rdimm = (lmc_control.s.rdimm_ena != 0); + // HACK, should do better + is_1slot = (lmc_config.s.init_status < 4); + + ddr_min_speed = ddr_speed_filter[is_ddr4][is_rdimm][is_1slot]; + do_tune = ((ddr_min_speed != 0) && (ddr_speed > ddr_min_speed)); + + debug("N%d: DDR%d %cDIMM %d-slot at %d MHz %s eligible for auto-tuning.\n", + node, (is_ddr4) ? 4 : 3, (is_rdimm) ? 'R' : 'U', + (is_1slot) ? 1 : 2, ddr_speed, (do_tune) ? "is" : "is not"); + + // call the tuning routine, filtering is done... + if (do_tune) + cvmx_tune_node(priv); +} + +/* + * first pattern example: + * GENERAL_PURPOSE0.DATA == 64'h00ff00ff00ff00ff; + * GENERAL_PURPOSE1.DATA == 64'h00ff00ff00ff00ff; + * GENERAL_PURPOSE0.DATA == 16'h0000; + */ + +static const u64 dbi_pattern[3] = { + 0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL, 0x0000ULL }; + +// Perform switchover to DBI +static void cvmx_dbi_switchover_interface(struct ddr_priv *priv, int lmc) +{ + union cvmx_lmcx_modereg_params0 modereg_params0; + union cvmx_lmcx_modereg_params3 modereg_params3; + union cvmx_lmcx_phy_ctl phy_ctl; + union cvmx_lmcx_config lmcx_config; + union cvmx_lmcx_ddr_pll_ctl ddr_pll_ctl; + int rank_mask, rankx, active_ranks; + u64 phys_addr, rank_offset; + int num_lmcs, errors; + int dbi_settings[9], byte, unlocked, retries; + int ecc_ena; + int rank_max = 1; // FIXME: make this 4 to try all the ranks + int node = 0; + + ddr_pll_ctl.u64 = lmc_rd(priv, CVMX_LMCX_DDR_PLL_CTL(0)); + + lmcx_config.u64 = lmc_rd(priv, CVMX_LMCX_CONFIG(lmc)); + rank_mask = lmcx_config.s.init_status; + ecc_ena = lmcx_config.s.ecc_ena; + + // FIXME: must filter out any non-supported configs + // ie, no DDR3, no x4 devices + if (ddr_pll_ctl.s.ddr4_mode == 0 || lmcx_config.s.mode_x4dev == 1) { + debug("N%d.LMC%d: DBI switchover: inappropriate device; EXITING...\n", + node, lmc); + return; + } + + // this should be correct for 1 or 2 ranks, 1 or 2 DIMMs + num_lmcs = cvmx_dram_get_num_lmc(priv); + rank_offset = 1ull << (28 + lmcx_config.s.pbank_lsb - + lmcx_config.s.rank_ena + (num_lmcs / 2)); + + debug("N%d.LMC%d: DBI switchover: rank mask 0x%x, rank size 0x%016llx.\n", + node, lmc, rank_mask, (unsigned long long)rank_offset); + + /* + * 1. conduct the current init sequence as usual all the way + * after software write leveling. + */ + + read_dac_dbi_settings(priv, lmc, /*DBI*/ 0, dbi_settings); + + display_dac_dbi_settings(lmc, /*DBI*/ 0, ecc_ena, dbi_settings, + " INIT"); + + /* + * 2. set DBI related CSRs as below and issue MR write. + * MODEREG_PARAMS3.WR_DBI=1 + * MODEREG_PARAMS3.RD_DBI=1 + * PHY_CTL.DBI_MODE_ENA=1 + */ + modereg_params0.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS0(lmc)); + + modereg_params3.u64 = lmc_rd(priv, CVMX_LMCX_MODEREG_PARAMS3(lmc)); + modereg_params3.s.wr_dbi = 1; + modereg_params3.s.rd_dbi = 1; + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS3(lmc), modereg_params3.u64); + + phy_ctl.u64 = lmc_rd(priv, CVMX_LMCX_PHY_CTL(lmc)); + phy_ctl.s.dbi_mode_ena = 1; + lmc_wr(priv, CVMX_LMCX_PHY_CTL(lmc), phy_ctl.u64); + + /* + * there are two options for data to send. Lets start with (1) + * and could move to (2) in the future: + * + * 1) DBTRAIN_CTL[LFSR_PATTERN_SEL] = 0 (or for older chips where + * this does not exist) set data directly in these reigsters. + * this will yield a clk/2 pattern: + * GENERAL_PURPOSE0.DATA == 64'h00ff00ff00ff00ff; + * GENERAL_PURPOSE1.DATA == 64'h00ff00ff00ff00ff; + * GENERAL_PURPOSE0.DATA == 16'h0000; + * 2) DBTRAIN_CTL[LFSR_PATTERN_SEL] = 1 + * here data comes from the LFSR generating a PRBS pattern + * CHAR_CTL.EN = 0 + * CHAR_CTL.SEL = 0; // for PRBS + * CHAR_CTL.DR = 1; + * CHAR_CTL.PRBS = setup for whatever type of PRBS to send + * CHAR_CTL.SKEW_ON = 1; + */ + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE0(lmc), dbi_pattern[0]); + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE1(lmc), dbi_pattern[1]); + lmc_wr(priv, CVMX_LMCX_GENERAL_PURPOSE2(lmc), dbi_pattern[2]); + + /* + * 3. adjust cas_latency (only necessary if RD_DBI is set). + * here is my code for doing this: + * + * if (csr_model.MODEREG_PARAMS3.RD_DBI.value == 1) begin + * case (csr_model.MODEREG_PARAMS0.CL.value) + * 0,1,2,3,4: csr_model.MODEREG_PARAMS0.CL.value += 2; + * // CL 9-13 -> 11-15 + * 5: begin + * // CL=14, CWL=10,12 gets +2, CLW=11,14 gets +3 + * if((csr_model.MODEREG_PARAMS0.CWL.value==1 || + * csr_model.MODEREG_PARAMS0.CWL.value==3)) + * csr_model.MODEREG_PARAMS0.CL.value = 7; // 14->16 + * else + * csr_model.MODEREG_PARAMS0.CL.value = 13; // 14->17 + * end + * 6: csr_model.MODEREG_PARAMS0.CL.value = 8; // 15->18 + * 7: csr_model.MODEREG_PARAMS0.CL.value = 14; // 16->19 + * 8: csr_model.MODEREG_PARAMS0.CL.value = 15; // 18->21 + * default: + * `cn_fatal(("Error mem_cfg (%s) CL (%d) with RD_DBI=1, + * I am not sure what to do.", + * mem_cfg, csr_model.MODEREG_PARAMS3.RD_DBI.value)) + * endcase + * end + */ + + if (modereg_params3.s.rd_dbi == 1) { + int old_cl, new_cl, old_cwl; + + old_cl = modereg_params0.s.cl; + old_cwl = modereg_params0.s.cwl; + + switch (old_cl) { + case 0: + case 1: + case 2: + case 3: + case 4: + new_cl = old_cl + 2; + break; // 9-13->11-15 + // CL=14, CWL=10,12 gets +2, CLW=11,14 gets +3 + case 5: + new_cl = ((old_cwl == 1) || (old_cwl == 3)) ? 7 : 13; + break; + case 6: + new_cl = 8; + break; // 15->18 + case 7: + new_cl = 14; + break; // 16->19 + case 8: + new_cl = 15; + break; // 18->21 + default: + printf("ERROR: Bad CL value (%d) for DBI switchover.\n", + old_cl); + // FIXME: need to error exit here... + old_cl = -1; + new_cl = -1; + break; + } + debug("N%d.LMC%d: DBI switchover: CL ADJ: old_cl 0x%x, old_cwl 0x%x, new_cl 0x%x.\n", + node, lmc, old_cl, old_cwl, new_cl); + modereg_params0.s.cl = new_cl; + lmc_wr(priv, CVMX_LMCX_MODEREG_PARAMS0(lmc), + modereg_params0.u64); + } + + /* + * 4. issue MRW to MR0 (CL) and MR5 (DBI), using LMC sequence + * SEQ_CTL[SEQ_SEL] = MRW. + */ + // Use the default values, from the CSRs fields + // also, do B-sides for RDIMMs... + + for (rankx = 0; rankx < 4; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + // for RDIMMs, B-side writes should get done automatically + // when the A-side is written + ddr4_mrw(priv, lmc, rankx, -1 /* use_default */, + 0 /*MRreg */, 0 /*A-side */); /* MR0 */ + ddr4_mrw(priv, lmc, rankx, -1 /* use_default */, + 5 /*MRreg */, 0 /*A-side */); /* MR5 */ + } + + /* + * 5. conduct DBI bit deskew training via the General Purpose + * R/W sequence (dbtrain). may need to run this over and over to get + * a lock (I need up to 5 in simulation): + * SEQ_CTL[SEQ_SEL] = RW_TRAINING (15) + * DBTRAIN_CTL.CMD_COUNT_EXT = all 1's + * DBTRAIN_CTL.READ_CMD_COUNT = all 1's + * DBTRAIN_CTL.TCCD_SEL = set according to MODEREG_PARAMS3[TCCD_L] + * DBTRAIN_CTL.RW_TRAIN = 1 + * DBTRAIN_CTL.READ_DQ_COUNT = dont care + * DBTRAIN_CTL.WRITE_ENA = 1; + * DBTRAIN_CTL.ACTIVATE = 1; + * DBTRAIN_CTL LRANK, PRANK, ROW_A, BG, BA, COLUMN_A = set to a + * valid address + */ + + // NOW - do the training + debug("N%d.LMC%d: DBI switchover: TRAINING begins...\n", node, lmc); + + active_ranks = 0; + for (rankx = 0; rankx < rank_max; rankx++) { + if (!(rank_mask & (1 << rankx))) + continue; + + phys_addr = rank_offset * active_ranks; + // FIXME: now done by test_dram_byte_hw() + + active_ranks++; + + retries = 0; + +restart_training: + + // NOTE: return is a bitmask of the erroring bytelanes - + // we only print it + errors = + test_dram_byte_hw(priv, lmc, phys_addr, DBTRAIN_DBI, NULL); + + debug("N%d.LMC%d: DBI switchover: TEST: rank %d, phys_addr 0x%llx, errors 0x%x.\n", + node, lmc, rankx, (unsigned long long)phys_addr, errors); + + // NEXT - check for locking + unlocked = 0; + read_dac_dbi_settings(priv, lmc, /*DBI*/ 0, dbi_settings); + + for (byte = 0; byte < (8 + ecc_ena); byte++) + unlocked += (dbi_settings[byte] & 1) ^ 1; + + // FIXME: print out the DBI settings array after each rank? + if (rank_max > 1) // only when doing more than 1 rank + display_dac_dbi_settings(lmc, /*DBI*/ 0, ecc_ena, + dbi_settings, " RANK"); + + if (unlocked > 0) { + debug("N%d.LMC%d: DBI switchover: LOCK: %d still unlocked.\n", + node, lmc, unlocked); + retries++; + if (retries < 10) { + goto restart_training; + } else { + debug("N%d.LMC%d: DBI switchover: LOCK: %d retries exhausted.\n", + node, lmc, retries); + } + } + } /* for (rankx = 0; rankx < 4; rankx++) */ + + // print out the final DBI settings array + display_dac_dbi_settings(lmc, /*DBI*/ 0, ecc_ena, dbi_settings, + "FINAL"); +} + +void cvmx_dbi_switchover(struct ddr_priv *priv) +{ + int lmc; + int num_lmcs = cvmx_dram_get_num_lmc(priv); + + for (lmc = 0; lmc < num_lmcs; lmc++) + cvmx_dbi_switchover_interface(priv, lmc); +}