On Mon, Jun 8, 2020 at 1:48 PM Stefan Sperling <[email protected]> wrote:
>
> On Fri, May 22, 2020 at 01:48:28PM -0400, sven falempin wrote:
> > After a few days ... (free size too small 288 < 1024 /2 )
> >
> > Maybe this can help make the driver better.
> >
> > printf '%x\n' $((0x350+0xf7)) ; grep -A2 'if_iwx.c:515' /tmp/iwx.dis
> > 447
> > /usr/src/sys/dev/pci/if_iwx.c:515
> > 447: 41 c7 86 28 2f 05 00 movl $0x0,0x52f28(%r14)
> > 44e: 00 00 00 00
> >
> > [0]-[current]-[~]
> > # cat -n /usr/src/sys/dev/pci/if_iwx.c | grep -C5 -E ' 515'
> > 510 /* free paging*/
> > 511 for (i = 0; i < dram->paging_cnt; i++)
> > 512 iwx_dma_contig_free(dram->paging);
> > 513
> > 514 free(dram->paging, M_DEVBUF, dram->paging_cnt *
> > sizeof(*dram->paging));
> > 515 dram->paging_cnt = 0;
> > 516 dram->paging = NULL;
> > 517 }
> > 518
> > 519 int
> > 520 iwx_get_num_sections(const struct iwx_fw_sects *fws, int start
>
> This should fix free with a wrong size in the error case, and avoids
> re-allocating a chunk of DMA memory (sc->ctxt_info_dma) every time the
> firmware gets loaded. Instead, this chunk is now allocated once at
> attach time. This seems to be the allocation that failed in your case.
>
> diff 66ecf2e2f524653126dce17a447a43b26ee90abb /usr/src
> blob - c3ca08c7a726326e37cda8645596a176051b6cf4
> file + sys/dev/pci/if_iwx.c
> --- sys/dev/pci/if_iwx.c
> +++ sys/dev/pci/if_iwx.c
> @@ -230,7 +230,7 @@ int iwx_alloc_fw_monitor_block(struct iwx_softc *, uin
> int iwx_alloc_fw_monitor(struct iwx_softc *, uint8_t);
> int iwx_apply_debug_destination(struct iwx_softc *);
> int iwx_ctxt_info_init(struct iwx_softc *, const struct iwx_fw_sects *);
> -void iwx_ctxt_info_free(struct iwx_softc *);
> +void iwx_ctxt_info_free_fw_img(struct iwx_softc *);
> void iwx_ctxt_info_free_paging(struct iwx_softc *);
> int iwx_init_fw_sec(struct iwx_softc *, const struct iwx_fw_sects *,
> struct iwx_context_info_dram *);
> @@ -535,52 +535,60 @@ iwx_init_fw_sec(struct iwx_softc *sc, const struct iwx
> struct iwx_context_info_dram *ctxt_dram)
> {
> struct iwx_self_init_dram *dram = &sc->init_dram;
> - int i, ret, lmac_cnt, umac_cnt, paging_cnt;
> + int i, ret, fw_cnt = 0;
>
> KASSERT(dram->paging == NULL);
>
> - lmac_cnt = iwx_get_num_sections(fws, 0);
> + dram->lmac_cnt = iwx_get_num_sections(fws, 0);
> /* add 1 due to separator */
> - umac_cnt = iwx_get_num_sections(fws, lmac_cnt + 1);
> + dram->umac_cnt = iwx_get_num_sections(fws, dram->lmac_cnt + 1);
> /* add 2 due to separators */
> - paging_cnt = iwx_get_num_sections(fws, lmac_cnt + umac_cnt + 2);
> + dram->paging_cnt = iwx_get_num_sections(fws,
> + dram->lmac_cnt + dram->umac_cnt + 2);
>
> - dram->fw = mallocarray(umac_cnt + lmac_cnt, sizeof(*dram->fw),
> - M_DEVBUF, M_ZERO | M_NOWAIT);
> - if (!dram->fw)
> + dram->fw = mallocarray(dram->umac_cnt + dram->lmac_cnt,
> + sizeof(*dram->fw), M_DEVBUF, M_ZERO | M_NOWAIT);
> + if (!dram->fw) {
> + printf("%s: could not allocate memory for firmware
> sections\n",
> + DEVNAME(sc));
> return ENOMEM;
> - dram->paging = mallocarray(paging_cnt, sizeof(*dram->paging),
> + }
> +
> + dram->paging = mallocarray(dram->paging_cnt, sizeof(*dram->paging),
> M_DEVBUF, M_ZERO | M_NOWAIT);
> - if (!dram->paging)
> + if (!dram->paging) {
> + printf("%s: could not allocate memory for firmware paging\n",
> + DEVNAME(sc));
> return ENOMEM;
> + }
>
> /* initialize lmac sections */
> - for (i = 0; i < lmac_cnt; i++) {
> + for (i = 0; i < dram->lmac_cnt; i++) {
> ret = iwx_ctxt_info_alloc_dma(sc, &fws->fw_sect[i],
> - &dram->fw[dram->fw_cnt]);
> + &dram->fw[fw_cnt]);
> if (ret)
> return ret;
> ctxt_dram->lmac_img[i] =
> - htole64(dram->fw[dram->fw_cnt].paddr);
> + htole64(dram->fw[fw_cnt].paddr);
> DPRINTF(("%s: firmware LMAC section %d at 0x%llx size
> %lld\n", __func__, i,
> - (unsigned long long)dram->fw[dram->fw_cnt].paddr,
> - (unsigned long long)dram->fw[dram->fw_cnt].size));
> - dram->fw_cnt++;
> + (unsigned long long)dram->fw[fw_cnt].paddr,
> + (unsigned long long)dram->fw[fw_cnt].size));
> + fw_cnt++;
> }
>
> /* initialize umac sections */
> - for (i = 0; i < umac_cnt; i++) {
> + for (i = 0; i < dram->umac_cnt; i++) {
> /* access FW with +1 to make up for lmac separator */
> ret = iwx_ctxt_info_alloc_dma(sc,
> - &fws->fw_sect[dram->fw_cnt + 1], &dram->fw[dram->fw_cnt]);
> + &fws->fw_sect[fw_cnt + 1], &dram->fw[fw_cnt]);
> if (ret)
> return ret;
> ctxt_dram->umac_img[i] =
> - htole64(dram->fw[dram->fw_cnt].paddr);
> + htole64(dram->fw[fw_cnt].paddr);
> DPRINTF(("%s: firmware UMAC section %d at 0x%llx size
> %lld\n", __func__, i,
> - (unsigned long long)dram->fw[dram->fw_cnt].paddr,
> - (unsigned long long)dram->fw[dram->fw_cnt].size));
> - dram->fw_cnt++;
> + (unsigned long long)dram->fw[fw_cnt].paddr,
> + (unsigned long long)dram->fw[fw_cnt].size));
> + fw_cnt++;
> }
>
> /*
> @@ -593,9 +601,9 @@ iwx_init_fw_sec(struct iwx_softc *sc, const struct iwx
> * Given that, the logic here in accessing the fw image is a bit
> * different - fw_cnt isn't changing so loop counter is added to it.
> */
> - for (i = 0; i < paging_cnt; i++) {
> + for (i = 0; i < dram->paging_cnt; i++) {
> /* access FW with +2 to make up for lmac & umac separators */
> - int fw_idx = dram->fw_cnt + i + 2;
> + int fw_idx = fw_cnt + i + 2;
>
> ret = iwx_ctxt_info_alloc_dma(sc,
> &fws->fw_sect[fw_idx], &dram->paging[i]);
> @@ -606,7 +614,6 @@ iwx_init_fw_sec(struct iwx_softc *sc, const struct iwx
> DPRINTF(("%s: firmware paging section %d at 0x%llx size
> %lld\n", __func__, i,
> (unsigned long long)dram->paging[i].paddr,
> (unsigned long long)dram->paging[i].size));
> - dram->paging_cnt++;
> }
>
> return 0;
> @@ -758,13 +765,6 @@ iwx_ctxt_info_init(struct iwx_softc *sc, const struct
> uint32_t control_flags = 0, rb_size;
> int err;
>
> - err = iwx_dma_contig_alloc(sc->sc_dmat, &sc->ctxt_info_dma,
> - sizeof(*ctxt_info), 0);
> - if (err) {
> - printf("%s: could not allocate context info DMA memory\n",
> - DEVNAME(sc));
> - return err;
> - }
> ctxt_info = sc->ctxt_info_dma.vaddr;
>
> ctxt_info->version.version = 0;
> @@ -800,15 +800,17 @@ iwx_ctxt_info_init(struct iwx_softc *sc, const struct
> /* allocate ucode sections in dram and set addresses */
> err = iwx_init_fw_sec(sc, fws, &ctxt_info->dram);
> if (err) {
> - iwx_ctxt_info_free(sc);
> + iwx_ctxt_info_free_fw_img(sc);
> return err;
> }
>
> /* Configure debug, if exists */
> if (sc->sc_fw.dbg_dest_tlv_v1) {
> err = iwx_apply_debug_destination(sc);
> - if (err)
> + if (err) {
> + iwx_ctxt_info_free_fw_img(sc);
> return err;
> + }
> }
>
> /* kick FW self load */
> @@ -829,26 +831,19 @@ iwx_ctxt_info_free_fw_img(struct iwx_softc *sc)
> struct iwx_self_init_dram *dram = &sc->init_dram;
> int i;
>
> - if (!dram->fw) {
> - KASSERT(dram->fw_cnt == 0);
> + if (!dram->fw)
> return;
> - }
>
> - for (i = 0; i < dram->fw_cnt; i++)
> + for (i = 0; i < dram->lmac_cnt + dram->umac_cnt; i++)
> iwx_dma_contig_free(&dram->fw[i]);
>
> - free(dram->fw, M_DEVBUF, dram->fw_cnt * sizeof(dram->fw[0]));
> - dram->fw_cnt = 0;
> + free(dram->fw, M_DEVBUF,
> + (dram->lmac_cnt + dram->umac_cnt) * sizeof(*dram->fw));
> + dram->lmac_cnt = 0;
> + dram->umac_cnt = 0;
> dram->fw = NULL;
> }
>
> -void
> -iwx_ctxt_info_free(struct iwx_softc *sc)
> -{
> - iwx_dma_contig_free(&sc->ctxt_info_dma);
> - iwx_ctxt_info_free_fw_img(sc);
> -}
> -
> int
> iwx_firmware_store_section(struct iwx_softc *sc, enum iwx_ucode_type type,
> uint8_t *data, size_t dlen)
> @@ -2410,7 +2405,6 @@ void
> iwx_post_alive(struct iwx_softc *sc)
> {
> iwx_ict_reset(sc);
> - iwx_ctxt_info_free(sc);
> }
>
> /*
> @@ -3113,6 +3107,9 @@ iwx_load_firmware(struct iwx_softc *sc)
> }
> if (err || !sc->sc_uc.uc_ok)
> printf("%s: could not load firmware\n", DEVNAME(sc));
> +
> + iwx_ctxt_info_free_fw_img(sc);
> +
> if (!sc->sc_uc.uc_ok)
> return EINVAL;
>
> @@ -7774,6 +7771,15 @@ iwx_attach(struct device *parent, struct device *self,
> return;
> }
>
> + /* Allocate DMA memory for loading firmware. */
> + err = iwx_dma_contig_alloc(sc->sc_dmat, &sc->ctxt_info_dma,
> + sizeof(struct iwx_context_info), 0);
> + if (err) {
> + printf("%s: could not allocate memory for loading firmware\n",
> + DEVNAME(sc));
> + return;
> + }
> +
> /*
> * Allocate DMA memory for firmware transfers.
> * Must be aligned on a 16-byte boundary.
> @@ -7781,9 +7787,9 @@ iwx_attach(struct device *parent, struct device *self,
> err = iwx_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
> sc->sc_fwdmasegsz, 16);
> if (err) {
> - printf("%s: could not allocate memory for firmware\n",
> + printf("%s: could not allocate memory for firmware
> transfers\n",
> DEVNAME(sc));
> - return;
> + goto fail0;
> }
>
> /* Allocate interrupt cause table (ICT).*/
> @@ -7915,6 +7921,7 @@ fail3: if (sc->ict_dma.vaddr != NULL)
> iwx_dma_contig_free(&sc->ict_dma);
>
> fail1: iwx_dma_contig_free(&sc->fw_dma);
> +fail0: iwx_dma_contig_free(&sc->ctxt_info_dma);
> return;
> }
>
> blob - 7a0d1c61f771b3691baa72ea835940bcffb48c64
> file + sys/dev/pci/if_iwxvar.h
> --- sys/dev/pci/if_iwxvar.h
> +++ sys/dev/pci/if_iwxvar.h
> @@ -346,13 +346,15 @@ struct iwx_bf_data {
> /**
> * struct iwx_self_init_dram - dram data used by self init process
> * @fw: lmac and umac dram data
> - * @fw_cnt: total number of items in array
> + * @lmac_cnt: number of lmac sections in fw image
> + * @umac_cnt: number of umac sections in fw image
> * @paging: paging dram data
> - * @paging_cnt: total number of items in array
> + * @paging_cnt: number of paging sections needed by fw image
> */
> struct iwx_self_init_dram {
> struct iwx_dma_info *fw;
> - int fw_cnt;
> + int lmac_cnt;
> + int umac_cnt;
> struct iwx_dma_info *paging;
> int paging_cnt;
> };
applied on current, for the sake of it, I ran the reboot sequence (1)
Note that even when firmware `boot`, scanning may trigger error but
non permanent.
I may have to reboot the device but i ll try keeping it on to see if
the DMA error is triggered
//scan ok, scan ok .. then
# ifconfig iwx0 scan; syslogc -c messages
iwx0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
lladdr f8:e4:e3:23:3c:46
index 1 priority 4 llprio 3
groups: wlan
media: IEEE802.11 autoselect
status: no network
ieee80211: nwid ""
none
Jun 8 19:54:33 current /bsd: iwx0: dumping device error log
Jun 8 19:54:33 current /bsd: iwx0: Start Error Log Dump:
Jun 8 19:54:33 current /bsd: iwx0: Status: 0x5, count: 6
Jun 8 19:54:33 current /bsd: iwx0: 0x00002073 | ADVANCED_SYSASSERT
Jun 8 19:54:33 current /bsd: iwx0: 000022F0 | trm_hw_status0
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | trm_hw_status1
Jun 8 19:54:33 current /bsd: iwx0: 004FC308 | branchlink2
Jun 8 19:54:33 current /bsd: iwx0: 004E8926 | interruptlink1
Jun 8 19:54:33 current /bsd: iwx0: 004E8926 | interruptlink2
Jun 8 19:54:33 current /bsd: iwx0: 00000001 | data1
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | data2
Jun 8 19:54:33 current /bsd: iwx0: DEADBEEF | data3
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | beacon time
Jun 8 19:54:33 current /bsd: iwx0: 057B6CFB | tsf low
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | tsf hi
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | time gp1
Jun 8 19:54:33 current /bsd: iwx0: 057B6CFC | time gp2
Jun 8 19:54:33 current /bsd: iwx0: 00000001 | uCode revision type
Jun 8 19:54:33 current /bsd: iwx0: 0000002E | uCode version major
Jun 8 19:54:33 current /bsd: iwx0: 177B3E46 | uCode version minor
Jun 8 19:54:33 current /bsd: iwx0: 00000340 | hw version
Jun 8 19:54:33 current /bsd: iwx0: 18889000 | board version
Jun 8 19:54:33 current /bsd: iwx0: 80E2FC07 | hcmd
Jun 8 19:54:33 current /bsd: iwx0: 00020000 | isr0
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | isr1
Jun 8 19:54:33 current /bsd: iwx0: 08F00002 | isr2
Jun 8 19:54:33 current /bsd: iwx0: 00C0940C | isr3
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | isr4
Jun 8 19:54:33 current /bsd: iwx0: 000A00D2 | last cmd Id
Jun 8 19:54:33 current /bsd: iwx0: 00017240 | wait_event
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | l2p_control
Jun 8 19:54:33 current /bsd: iwx0: 00000020 | l2p_duration
Jun 8 19:54:33 current /bsd: iwx0: 00000000 | l2p_mhvalid
Jun 8 19:54:33 current /bsd: iwx0: 00000080 | l2p_addr_match
Jun 8 19:54:33 current /bsd: iwx0: 00000009 | lmpm_pmg_sel
Jun 8 19:54:33 current /bsd: iwx0: 19071335 | timestamp
Jun 8 19:54:33 current /bsd: iwx0: 0000A8CC | flow_handler
Jun 8 19:54:33 current /bsd: iwx0: Start UMAC Error Log Dump:
Jun 8 19:54:33 current /bsd: iwx0: Status: 0x5, count: 7
Jun 8 19:54:33 current /bsd: iwx0: 0x20000070 | NMI_INTERRUPT_LMAC_FATAL
Jun 8 19:54:33 current /bsd: iwx0: 0x00000000 | umac branchlink1
Jun 8 19:54:33 current /bsd: iwx0: 0xC008B1C0 | umac branchlink2
Jun 8 19:54:33 current /bsd: iwx0: 0x80489F5E | umac interruptlink1
Jun 8 19:54:33 current /bsd: iwx0: 0x80489F5E | umac interruptlink2
Jun 8 19:54:33 current /bsd: iwx0: 0x00000400 | umac data1
Jun 8 19:54:33 current /bsd: iwx0: 0x80489F5E | umac data2
Jun 8 19:54:33 current /bsd: iwx0: 0xDEADBEEF | umac data3
Jun 8 19:54:33 current /bsd: iwx0: 0x0000002E | umac major
Jun 8 19:54:33 current /bsd: iwx0: 0x177B3E46 | umac minor
Jun 8 19:54:33 current /bsd: iwx0: 0x057B6D0B | frame pointer
Jun 8 19:54:33 current /bsd: iwx0: 0xC088628C | stack pointer
Jun 8 19:54:33 current /bsd: iwx0: 0x00040018 | last host cmd
Jun 8 19:54:33 current /bsd: iwx0: 0x00000000 | isr status reg
Jun 8 19:54:33 current /bsd: driver status:
Jun 8 19:54:33 current /bsd: tx ring 0: qid=0 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 1: qid=1 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 2: qid=2 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 3: qid=3 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 4: qid=4 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 5: qid=5 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 6: qid=6 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 7: qid=7 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 8: qid=8 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 9: qid=9 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 10: qid=10 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 11: qid=11 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 12: qid=12 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 13: qid=13 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 14: qid=14 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 15: qid=15 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 16: qid=16 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 17: qid=17 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 18: qid=18 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 19: qid=19 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 20: qid=20 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 21: qid=21 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 22: qid=22 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 23: qid=23 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 24: qid=24 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 25: qid=25 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 26: qid=26 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 27: qid=27 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 28: qid=28 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 29: qid=29 cur=0 queued=0
Jun 8 19:54:33 current /bsd: tx ring 30: qid=30 cur=0 queued=0
Jun 8 19:54:33 current /bsd: rx ring: cur=176
Jun 8 19:54:33 current /bsd: 802.11 state SCAN
Jun 8 19:54:33 current /bsd: iwx0: fatal firmware error
(1)
FAILED Mon Jun 8 19:09:37 EDT 2020 1591657777
OK Mon Jun 8 19:10:19 EDT 2020 1591657819
OK Mon Jun 8 19:10:59 EDT 2020 1591657859
FAILED Mon Jun 8 19:11:43 EDT 2020 1591657903
OK Mon Jun 8 19:12:25 EDT 2020 1591657945
FAILED Mon Jun 8 19:13:09 EDT 2020 1591657989
FAILED Mon Jun 8 19:13:51 EDT 2020 1591658031
FAILED Mon Jun 8 19:14:34 EDT 2020 1591658074
OK Mon Jun 8 19:15:14 EDT 2020 1591658114
FAILED Mon Jun 8 19:15:58 EDT 2020 1591658158
OK Mon Jun 8 19:16:40 EDT 2020 1591658200
FAILED Mon Jun 8 19:17:24 EDT 2020 1591658244
FAILED Mon Jun 8 19:18:08 EDT 2020 1591658288
FAILED Mon Jun 8 19:18:52 EDT 2020 1591658332
OK Mon Jun 8 19:19:34 EDT 2020 1591658374
FAILED Mon Jun 8 19:20:18 EDT 2020 1591658418
FAILED Mon Jun 8 19:21:02 EDT 2020 1591658462
OK Mon Jun 8 19:21:44 EDT 2020 1591658504
OK Mon Jun 8 19:22:24 EDT 2020 1591658544
FAILED Mon Jun 8 19:23:08 EDT 2020 1591658588
OK Mon Jun 8 19:23:48 EDT 2020 1591658628
OK Mon Jun 8 19:24:30 EDT 2020 1591658670
FAILED Mon Jun 8 19:25:14 EDT 2020 1591658714
OK Mon Jun 8 19:25:54 EDT 2020 1591658754
FAILED Mon Jun 8 19:26:39 EDT 2020 1591658799
FAILED Mon Jun 8 19:27:23 EDT 2020 1591658843
FAILED Mon Jun 8 19:28:07 EDT 2020 1591658887
OK Mon Jun 8 19:28:47 EDT 2020 1591658927
OK Mon Jun 8 19:29:27 EDT 2020 1591658967
OK Mon Jun 8 19:30:07 EDT 2020 1591659007
FAILED Mon Jun 8 19:30:51 EDT 2020 1591659051
FAILED Mon Jun 8 19:31:35 EDT 2020 1591659095
FAILED Mon Jun 8 19:32:19 EDT 2020 1591659139
OK Mon Jun 8 19:32:59 EDT 2020 1591659179
FAILED Mon Jun 8 19:33:58 EDT 2020 1591659238
OK Mon Jun 8 19:34:38 EDT 2020 1591659278
OK Mon Jun 8 19:35:20 EDT 2020 1591659320
FAILED Mon Jun 8 19:36:04 EDT 2020 1591659364
FAILED Mon Jun 8 19:36:48 EDT 2020 1591659408
OK Mon Jun 8 19:37:28 EDT 2020 1591659448
OK Mon Jun 8 19:38:10 EDT 2020 1591659490
FAILED Mon Jun 8 19:38:54 EDT 2020 1591659534
FAILED Mon Jun 8 19:39:38 EDT 2020 1591659578
FAILED Mon Jun 8 19:40:22 EDT 2020 1591659622
OK Mon Jun 8 19:41:02 EDT 2020 1591659662
OK Mon Jun 8 19:41:42 EDT 2020 1591659702
OK Mon Jun 8 19:42:22 EDT 2020 1591659742
FAILED Mon Jun 8 19:43:06 EDT 2020 1591659786
FAILED Mon Jun 8 19:43:50 EDT 2020 1591659830
--
--
---------------------------------------------------------------------------------------------------------------------
Knowing is not enough; we must apply. Willing is not enough; we must do
