From 6bc6676ec4a1029057c9be9cdbdeb7e3b9e6f757 Mon Sep 17 00:00:00 2001 From: mtk81325 Date: Thu, 28 Jan 2021 19:45:06 +0800 Subject: [PATCH] [ALPS04936086] BLOCKTAG: IO state extension Its a legacy feature of IO trace in MTK platform. Change: - MQ, CMDQD changed to worker(HW dispatch queue) ToDo: - decouple with debugfs - distinguish emmc and SD - vmstat support (mdlog had been removed) MTK-Commit-Id: 42b0bd45cc6e7e096959be88fce8ed38d13854c3 Change-Id: Ie78cad95a0884d0f6eda2045f27f647e070705ae Signed-off-by: mtk81325 CR-Id: ALPS04936086 Feature: [Android Default] F2FS File System --- block/Makefile | 2 +- block/blk-core.c | 7 +- block/blk-merge.c | 7 +- block/blk-mq.c | 3 + drivers/misc/mediatek/Kconfig | 2 +- drivers/misc/mediatek/Makefile | 1 + drivers/misc/mediatek/blocktag/Kconfig | 19 + drivers/misc/mediatek/blocktag/Makefile | 6 + drivers/misc/mediatek/blocktag/blocktag.c | 1364 +++++++++++++++++ .../mediatek/include/mt-plat/mtk_blocktag.h | 262 ++++ drivers/mmc/core/Kconfig | 13 + drivers/mmc/core/Makefile | 1 + drivers/mmc/core/block.c | 6 +- drivers/mmc/core/core.c | 22 +- drivers/mmc/core/mtk_mmc_block.c | 915 +++++++++++ drivers/mmc/core/mtk_mmc_block.h | 108 ++ drivers/mmc/core/queue.c | 2 + mm/page-writeback.c | 8 + 18 files changed, 2733 insertions(+), 15 deletions(-) create mode 100644 drivers/misc/mediatek/blocktag/Kconfig create mode 100644 drivers/misc/mediatek/blocktag/Makefile create mode 100644 drivers/misc/mediatek/blocktag/blocktag.c create mode 100644 drivers/misc/mediatek/include/mt-plat/mtk_blocktag.h create mode 100644 drivers/mmc/core/mtk_mmc_block.c create mode 100644 drivers/mmc/core/mtk_mmc_block.h diff --git a/block/Makefile b/block/Makefile index a2e05332682f..e7a3dacc69e1 100644 --- a/block/Makefile +++ b/block/Makefile @@ -2,7 +2,7 @@ # # Makefile for the kernel block layer # - +ccflags-y += -I$(srctree)/drivers/mmc/core obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ blk-flush.o blk-settings.o blk-ioc.o blk-map.o \ blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \ diff --git a/block/blk-core.c b/block/blk-core.c index f61a9f139cf8..502d1c375fed 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -37,6 +37,7 @@ #include #include #include +#include /* MTK PATCH */ #define CREATE_TRACE_POINTS #include @@ -45,6 +46,7 @@ #include "blk-mq.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" +#include "mtk_mmc_block.h" #ifdef CONFIG_DEBUG_FS struct dentry *blk_debugfs_root; @@ -413,7 +415,7 @@ void blk_sync_queue(struct request_queue *q) if (q->mq_ops) { struct blk_mq_hw_ctx *hctx; int i; - + mt_bio_queue_free(current); queue_for_each_hw_ctx(q, hctx, i) cancel_delayed_work_sync(&hctx->run_work); } else { @@ -2571,6 +2573,9 @@ blk_qc_t submit_bio(struct bio *bio) count_vm_events(PGPGIN, count); } +#ifdef CONFIG_MTK_BLOCK_TAG + mtk_btag_pidlog_submit_bio(bio); +#endif if (unlikely(block_dump)) { char b[BDEVNAME_SIZE]; printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", diff --git a/block/blk-merge.c b/block/blk-merge.c index 3e32f6a08864..5e1b0e763033 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -9,6 +9,7 @@ #include #include +#include /* MTK PATCH */ #include "blk.h" @@ -417,9 +418,13 @@ static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, int cluster = blk_queue_cluster(q), nsegs = 0; for_each_bio(bio) - bio_for_each_segment(bvec, bio, iter) + bio_for_each_segment(bvec, bio, iter) { __blk_segment_map_sg(q, &bvec, sglist, &bvprv, sg, &nsegs, &cluster); + #ifdef CONFIG_MTK_BLOCK_TAG + mtk_btag_pidlog_map_sg(q, bio, &bvec); + #endif + } return nsegs; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 684acaa96db7..1daf22359f19 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -36,6 +36,7 @@ #include "blk-stat.h" #include "blk-mq-sched.h" #include "blk-rq-qos.h" +#include "mtk_mmc_block.h" static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie); static void blk_mq_poll_stats_start(struct request_queue *q); @@ -1463,6 +1464,7 @@ EXPORT_SYMBOL(blk_mq_queue_stopped); */ void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) { + mt_bio_queue_free(current); cancel_delayed_work(&hctx->run_work); set_bit(BLK_MQ_S_STOPPED, &hctx->state); @@ -2186,6 +2188,7 @@ static int blk_mq_init_hctx(struct request_queue *q, if (node == NUMA_NO_NODE) node = hctx->numa_node = set->numa_node; + mt_bio_queue_alloc(current, q); INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); spin_lock_init(&hctx->lock); INIT_LIST_HEAD(&hctx->dispatch); diff --git a/drivers/misc/mediatek/Kconfig b/drivers/misc/mediatek/Kconfig index c83130a95251..b557cd23d99c 100644 --- a/drivers/misc/mediatek/Kconfig +++ b/drivers/misc/mediatek/Kconfig @@ -16,7 +16,7 @@ config MTK_ENG_BUILD If unsure, say N here. menu "Storage" - +source "drivers/misc/mediatek/blocktag/Kconfig" endmenu # Storage menu "Power, PMIC, Battery & Low Power" diff --git a/drivers/misc/mediatek/Makefile b/drivers/misc/mediatek/Makefile index fe26283fa724..a88a013bdb43 100644 --- a/drivers/misc/mediatek/Makefile +++ b/drivers/misc/mediatek/Makefile @@ -79,3 +79,4 @@ obj-$(CONFIG_RT_FLASHLIGHT) += flashlight/richtek/ obj-$(CONFIG_USB_MTK_HDRC) += usb20/ obj-$(CONFIG_MTK_USB_TYPEC) += typec/ obj-$(CONFIG_MTK_CCU) += ccu/src/ +obj-$(CONFIG_MTK_BLOCK_TAG) += blocktag/ diff --git a/drivers/misc/mediatek/blocktag/Kconfig b/drivers/misc/mediatek/blocktag/Kconfig new file mode 100644 index 000000000000..fa1d7d6bf579 --- /dev/null +++ b/drivers/misc/mediatek/blocktag/Kconfig @@ -0,0 +1,19 @@ +# +# block tag trace +# + +comment "Storage Block Tag" + +config MTK_BLOCK_TAG + bool "Storage Block Tag" + depends on BLOCK + depends on (MTK_GMO_RAM_OPTIMIZE && MTK_ENG_BUILD) || \ + !MTK_GMO_RAM_OPTIMIZE + depends on DEBUG_FS + help + Enable block tagging at block driver, tag requester pid to + the accessing pages. This allows MMC/UFS Block IO log to obtian + IO statistics of each process. The Block Tag also provides + utility functions to MMC/UFS Block IO log, such as throughput + calculation, log printing, and ring trace handling. + diff --git a/drivers/misc/mediatek/blocktag/Makefile b/drivers/misc/mediatek/blocktag/Makefile new file mode 100644 index 000000000000..c7ce7eb5d4c0 --- /dev/null +++ b/drivers/misc/mediatek/blocktag/Makefile @@ -0,0 +1,6 @@ +# +# Makefile for block tag drivers +# + +obj-$(CONFIG_MTK_BLOCK_TAG) += blocktag.o + diff --git a/drivers/misc/mediatek/blocktag/blocktag.c b/drivers/misc/mediatek/blocktag/blocktag.c new file mode 100644 index 000000000000..603e57897dca --- /dev/null +++ b/drivers/misc/mediatek/blocktag/blocktag.c @@ -0,0 +1,1364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 MediaTek Inc. + */ + +#define DEBUG 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BLOCKIO_MIN_VER "3.09" + +#ifdef CONFIG_MTK_USE_RESERVED_EXT_MEM +#include +#endif + +#include + +/* + * snprintf may return a value of size or "more" to indicate + * that the output was truncated, thus be careful of "more" + * case. + */ +#define SPREAD_PRINTF(buff, size, evt, fmt, args...) \ +do { \ + if (buff && size && *(size)) { \ + unsigned long var = snprintf(*(buff), *(size), fmt, ##args); \ + if (var > 0) { \ + if (var > *(size)) \ + var = *(size); \ + *(size) -= var; \ + *(buff) += var; \ + } \ + } \ + if (evt) \ + seq_printf(evt, fmt, ##args); \ + if (!buff && !evt) { \ + pr_info(fmt, ##args); \ + } \ +} while (0) + +#define mtk_btag_pidlog_index(p) \ + ((unsigned long)(__page_to_pfn(p)) - \ + (memblock_start_of_DRAM() >> PAGE_SHIFT)) + +#define mtk_btag_pidlog_max_entry() \ + (mtk_btag_system_dram_size >> PAGE_SHIFT) + +#define mtk_btag_pidlog_entry(idx) \ + (((struct page_pid_logger *)mtk_btag_pagelogger) + idx) + +/* max dump size is 300KB whitch can be adjusted */ +#define BLOCKIO_AEE_BUFFER_SIZE (300 * 1024) +char blockio_aee_buffer[BLOCKIO_AEE_BUFFER_SIZE]; + +/* debugfs dentries */ +struct dentry *mtk_btag_droot; +struct dentry *mtk_btag_dlog; + +/* mini context for major embedded storage only */ +#define MICTX_PROC_CMD_BUF_SIZE (1) +static struct mtk_btag_mictx_struct *mtk_btag_mictx; +static bool mtk_btag_mictx_ready; +static bool mtk_btag_mictx_debug; + +static void mtk_btag_init_debugfs(void); + +/* blocktag */ +static DEFINE_MUTEX(mtk_btag_list_lock); +static LIST_HEAD(mtk_btag_list); + +static struct mtk_blocktag *mtk_btag_find(const char *name) +{ + struct mtk_blocktag *btag, *n; + + list_for_each_entry_safe(btag, n, &mtk_btag_list, list) { + if (!strncmp(btag->name, name, BLOCKTAG_NAME_LEN-1)) + return btag; + } + return NULL; +} + +static struct mtk_blocktag *mtk_btag_find_locked(const char *name) +{ + struct mtk_blocktag *btag; + + mutex_lock(&mtk_btag_list_lock); + btag = mtk_btag_find(name); + mutex_unlock(&mtk_btag_list_lock); + return btag; +} + +/* pid logger: page loger*/ +unsigned long long mtk_btag_system_dram_size; +struct page_pid_logger *mtk_btag_pagelogger; + +static size_t mtk_btag_seq_pidlog_usedmem(char **buff, unsigned long *size, + struct seq_file *seq) +{ + size_t size_l = 0; + + if (!IS_ERR_OR_NULL(mtk_btag_pagelogger)) { + size_l = (sizeof(struct page_pid_logger) + * (mtk_btag_system_dram_size >> PAGE_SHIFT)); + SPREAD_PRINTF(buff, size, seq, + "page pid logger buffer: %llu entries * %zu = %zu bytes\n", + (mtk_btag_system_dram_size >> PAGE_SHIFT), + sizeof(struct page_pid_logger), + size_l); + } + return size_l; +} + +#define biolog_fmt "wl:%d%%,%lld,%lld,%d.vm:%lld,%lld,%lld,%lld,%lld,%lld." \ + "cpu:%llu,%llu,%llu,%llu,%llu,%llu,%llu.pid:%d," +#define biolog_fmt_wt "wt:%d,%d,%lld." +#define biolog_fmt_rt "rt:%d,%d,%lld." +#define pidlog_fmt "{%05d:%05d:%08d:%05d:%08d}" + +void mtk_btag_pidlog_insert(struct mtk_btag_pidlogger *pidlog, pid_t pid, + __u32 len, int write) +{ + int i; + struct mtk_btag_pidlogger_entry *pe; + struct mtk_btag_pidlogger_entry_rw *prw; + + for (i = 0; i < BLOCKTAG_PIDLOG_ENTRIES; i++) { + pe = &pidlog->info[i]; + if ((pe->pid == pid) || (pe->pid == 0)) { + pe->pid = pid; + prw = (write) ? &pe->w : &pe->r; + prw->count++; + prw->length += len; + break; + } + } +} +EXPORT_SYMBOL_GPL(mtk_btag_pidlog_insert); + +static void mtk_btag_pidlog_add(struct request_queue *q, struct bio *bio, + unsigned short pid, __u32 len) +{ + int write = bio_data_dir(bio); + int major = bio->bi_disk ? MAJOR(bio_dev(bio)) : 0; + + if (pid != 0xFFFF && major) { +#ifdef CONFIG_MTK_UFS_BLOCK_IO_LOG + if (major == SCSI_DISK0_MAJOR || major == BLOCK_EXT_MAJOR) { + mtk_btag_pidlog_add_ufs(q, pid, len, write); + return; + } +#endif +#ifdef CONFIG_MMC_BLOCK_IO_LOG + if (major == MMC_BLOCK_MAJOR || major == BLOCK_EXT_MAJOR) { + mtk_btag_pidlog_add_mmc(q, pid, len, write); + return; + } +#endif + } +} + +/* + * pidlog: hook function for __blk_bios_map_sg() + * rw: 0=read, 1=write + */ +void mtk_btag_pidlog_map_sg(struct request_queue *q, struct bio *bio, + struct bio_vec *bvec) +{ + struct page_pid_logger *ppl, tmp; + unsigned long idx; + + if (!mtk_btag_pagelogger || !bio || !bvec) + return; + + idx = mtk_btag_pidlog_index(bvec->bv_page); + ppl = mtk_btag_pidlog_entry(idx); + + tmp.pid = ppl->pid; + ppl->pid = 0xFFFF; + + mtk_btag_pidlog_add(q, bio, tmp.pid, bvec->bv_len); +} +EXPORT_SYMBOL_GPL(mtk_btag_pidlog_map_sg); + +static void _mtk_btag_pidlog_set_pid(struct page *p, int mode) +{ + struct page_pid_logger *ppl; + unsigned long idx; + + idx = mtk_btag_pidlog_index(p); + ppl = mtk_btag_pidlog_entry(idx); + + if (idx >= mtk_btag_pidlog_max_entry()) + return; + + /* we do lockless operation here to favor performance */ + + if (mode == PIDLOG_MODE_BLK_SUBMIT_BIO) { + /* + * do not overwrite the real owner set by + * mm or file system layer + */ + if (ppl->pid == 0xFFFF) + ppl->pid = current->pid; + } else { + /* the latest owner will be counted */ + ppl->pid = current->pid; + } +} + +void mtk_btag_pidlog_copy_pid(struct page *src, struct page *dst) +{ + struct page_pid_logger *ppl_src, *ppl_dst; + unsigned long idx_src, idx_dst; + + idx_src = mtk_btag_pidlog_index(src); + + if (idx_src >= mtk_btag_pidlog_max_entry()) + return; + + idx_dst = mtk_btag_pidlog_index(dst); + + if (idx_dst >= mtk_btag_pidlog_max_entry()) + return; + + ppl_src = mtk_btag_pidlog_entry(idx_src); + ppl_dst = mtk_btag_pidlog_entry(idx_dst); + ppl_dst->pid = ppl_src->pid; +} + +/* pidlog: hook function for submit_bio() */ +void mtk_btag_pidlog_submit_bio(struct bio *bio) +{ + struct bio_vec bvec; + struct bvec_iter iter; + + if (!mtk_btag_pagelogger) + return; + + bio_for_each_segment(bvec, bio, iter) { + if (bvec.bv_page) + _mtk_btag_pidlog_set_pid(bvec.bv_page, + PIDLOG_MODE_BLK_SUBMIT_BIO); + } +} +EXPORT_SYMBOL_GPL(mtk_btag_pidlog_submit_bio); + +void mtk_btag_pidlog_set_pid(struct page *p) +{ + if (!mtk_btag_pagelogger || !p) + return; + + _mtk_btag_pidlog_set_pid(p, PIDLOG_MODE_MM_FS); +} +EXPORT_SYMBOL_GPL(mtk_btag_pidlog_set_pid); + +/* evaluate vmstat trace from global_node_page_state() */ +void mtk_btag_vmstat_eval(struct mtk_btag_vmstat *vm) +{ + /* int cpu; */ + /* struct vm_event_state *this; */ + + vm->file_pages = ((global_node_page_state(NR_FILE_PAGES)) + << (PAGE_SHIFT - 10)); + vm->file_dirty = ((global_node_page_state(NR_FILE_DIRTY)) + << (PAGE_SHIFT - 10)); + vm->dirtied = ((global_node_page_state(NR_DIRTIED)) + << (PAGE_SHIFT - 10)); + vm->writeback = ((global_node_page_state(NR_WRITEBACK)) + << (PAGE_SHIFT - 10)); + vm->written = ((global_node_page_state(NR_WRITTEN)) + << (PAGE_SHIFT - 10)); + + /* file map fault */ + vm->fmflt = 0; +/* + * for_each_online_cpu(cpu) { + * this = &per_cpu(vm_event_states, cpu); + * vm->fmflt += this->event[PGFMFAULT]; + * } + */ +} +EXPORT_SYMBOL_GPL(mtk_btag_vmstat_eval); + +void mtk_btag_mictx_dump(void) +{ + struct mtk_btag_mictx_iostat_struct iostat; + int ret; + + ret = mtk_btag_mictx_get_data(&iostat); + + if (ret) { + pr_info("[BLOCK_TAG] Mictx: Get data failed %d\n", ret); + return; + } + + pr_info("[BLOCK_TAG] Mictx: %llu|%u|%u|%u|%u|%u|%u|%u|%u|%u|%u\n", + iostat.duration, iostat.q_depth, iostat.wl, + iostat.tp_req_r, iostat.tp_req_w, + iostat.tp_all_r, iostat.tp_all_w, + iostat.reqcnt_r, iostat.reqcnt_w, + iostat.reqsize_r, iostat.reqsize_w); +} +/* evaluate pidlog trace from context */ +void mtk_btag_pidlog_eval(struct mtk_btag_pidlogger *pl, + struct mtk_btag_pidlogger *ctx_pl) +{ + int i; + + for (i = 0; i < BLOCKTAG_PIDLOG_ENTRIES; i++) { + if (ctx_pl->info[i].pid == 0) + break; + } + + if (i != 0) { + int size = i * sizeof(struct mtk_btag_pidlogger_entry); + + memcpy(&pl->info[0], &ctx_pl->info[0], size); + memset(&ctx_pl->info[0], 0, size); + } + + if (mtk_btag_mictx_debug) + mtk_btag_mictx_dump(); +} +EXPORT_SYMBOL_GPL(mtk_btag_pidlog_eval); + +static __u64 mtk_btag_cpu_idle_time(int cpu) +{ + u64 idle, idle_usecs = -1ULL; + + if (cpu_online(cpu)) + idle_usecs = get_cpu_idle_time_us(cpu, NULL); + + if (idle_usecs == -1ULL) + /* !NO_HZ or cpu offline so we can rely on cpustat.idle */ + idle = kcpustat_cpu(cpu).cpustat[CPUTIME_IDLE]; + else + idle = idle_usecs * NSEC_PER_USEC; + + return idle; +} + +static __u64 mtk_btag_cpu_iowait_time(int cpu) +{ + __u64 iowait, iowait_usecs = -1ULL; + + if (cpu_online(cpu)) + iowait_usecs = get_cpu_iowait_time_us(cpu, NULL); + + if (iowait_usecs == -1ULL) + /* !NO_HZ or cpu offline so we can rely on cpustat.iowait */ + iowait = kcpustat_cpu(cpu).cpustat[CPUTIME_IOWAIT]; + else + iowait = iowait_usecs * NSEC_PER_USEC; + + return iowait; +} + +/* evaluate cpu trace from kcpustat_cpu() */ +void mtk_btag_cpu_eval(struct mtk_btag_cpu *cpu) +{ + int i; + __u64 user, nice, system, idle, iowait, irq, softirq; + + user = nice = system = idle = iowait = irq = softirq = 0; + + for_each_possible_cpu(i) { + user += kcpustat_cpu(i).cpustat[CPUTIME_USER]; + nice += kcpustat_cpu(i).cpustat[CPUTIME_NICE]; + system += kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]; + idle += mtk_btag_cpu_idle_time(i); + iowait += mtk_btag_cpu_iowait_time(i); + irq += kcpustat_cpu(i).cpustat[CPUTIME_IRQ]; + softirq += kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]; + } + + cpu->user = nsec_to_clock_t(user); + cpu->nice = nsec_to_clock_t(nice); + cpu->system = nsec_to_clock_t(system); + cpu->idle = nsec_to_clock_t(idle); + cpu->iowait = nsec_to_clock_t(iowait); + cpu->irq = nsec_to_clock_t(irq); + cpu->softirq = nsec_to_clock_t(softirq); +} +EXPORT_SYMBOL_GPL(mtk_btag_cpu_eval); + +static __u32 mtk_btag_eval_tp_speed(__u32 bytes, __u64 duration) +{ + __u32 speed_kbs = 0; + + if (!bytes || !duration) + return 0; + + /* convert ns to ms */ + do_div(duration, 1000000); + + if (duration) { + /* bytes/ms */ + speed_kbs = bytes / (__u32)duration; + + /* KB/s */ + speed_kbs = (speed_kbs * 1000) >> 10; + } + + return speed_kbs; +} + +static void mtk_btag_throughput_rw_eval(struct mtk_btag_throughput_rw *rw) +{ + __u64 usage; + + usage = rw->usage; + + do_div(usage, 1000000); /* convert ns to ms */ + + if (usage && rw->size) { + rw->speed = (rw->size) / (__u32)usage; /* bytes/ms */ + rw->speed = (rw->speed*1000) >> 10; /* KB/s */ + rw->usage = usage; + } else { + rw->speed = 0; + rw->size = 0; + rw->usage = 0; + } +} +/* calculate throughput */ +void mtk_btag_throughput_eval(struct mtk_btag_throughput *tp) +{ + mtk_btag_throughput_rw_eval(&tp->r); + mtk_btag_throughput_rw_eval(&tp->w); +} +EXPORT_SYMBOL_GPL(mtk_btag_throughput_eval); + +/* print trace to kerne log */ +static void mtk_btag_klog_entry(char **ptr, int *len, struct mtk_btag_trace *tr) +{ + int i, n; + +#define boundary_check() { *len -= n; *ptr += n; } + + if (tr->throughput.r.usage) { + n = snprintf(*ptr, *len, biolog_fmt_rt, + tr->throughput.r.speed, + tr->throughput.r.size, + tr->throughput.r.usage); + boundary_check(); + if (*len < 0) + return; + } + + if (tr->throughput.w.usage) { + n = snprintf(*ptr, *len, biolog_fmt_wt, + tr->throughput.w.speed, + tr->throughput.w.size, + tr->throughput.w.usage); + boundary_check(); + if (*len < 0) + return; + } + //if (!tr->throughput.w.usage || !tr->throughput.r.usage) + n = snprintf(*ptr, *len, biolog_fmt, + tr->workload.percent, + tr->workload.usage, + tr->workload.period, + tr->workload.count, + tr->vmstat.file_pages, + tr->vmstat.file_dirty, + tr->vmstat.dirtied, + tr->vmstat.writeback, + tr->vmstat.written, + tr->vmstat.fmflt, + tr->cpu.user, + tr->cpu.nice, + tr->cpu.system, + tr->cpu.idle, + tr->cpu.iowait, + tr->cpu.irq, + tr->cpu.softirq, + tr->pid); + boundary_check(); + if (*len < 0) + return; + + for (i = 0; i < BLOCKTAG_PIDLOG_ENTRIES; i++) { + struct mtk_btag_pidlogger_entry *pe; + + pe = &tr->pidlog.info[i]; + + if (pe->pid == 0) + break; + + n = snprintf(*ptr, *len, pidlog_fmt, + pe->pid, + pe->w.count, + pe->w.length, + pe->r.count, + pe->r.length); + boundary_check(); + if (*len < 0) + return; + } +} + +void mtk_btag_klog(struct mtk_blocktag *btag, struct mtk_btag_trace *tr) +{ + int len; + char *ptr; + unsigned long flags; + + if (!btag || !btag->klog_enable || !tr) + return; + + len = BLOCKTAG_PRINT_LEN-1; + ptr = &btag->prbuf.buf[0]; + + spin_lock_irqsave(&btag->prbuf.lock, flags); + mtk_btag_klog_entry(&ptr, &len, tr); + spin_unlock_irqrestore(&btag->prbuf.lock, flags); +} +EXPORT_SYMBOL_GPL(mtk_btag_klog); + +static int mtk_btag_pr_time(char *out, int size, const char *str, __u64 t) +{ + uint32_t nsec; + int ret; + + nsec = do_div(t, 1000000000); + ret = snprintf(out, size, ",%s=[%lu.%06lu]", str, (unsigned long)t, + (unsigned long)nsec/1000); + return ret; +} + +static const char *mtk_btag_pr_speed(char *out, int size, __u64 usage, + __u32 bytes) +{ + __u32 speed; + + if (!usage || !bytes) + return ""; + + do_div(usage, 1000); /* convert ns to us */ + speed = 1000 * bytes / (__u32)usage; /* bytes/ms */ + speed = (speed*1000) >> 10; /* KB/s */ + + snprintf(out, size, ",%u KB/s", speed); + return out; +} + +void mtk_btag_task_timetag(char *buf, unsigned int len, unsigned int stage, + unsigned int max, const char *name[], uint64_t *t, __u32 bytes) +{ + __u64 busy_time = 0; + int i; + int ret; + + if (!buf || !len) + return; + + for (i = 0; i <= stage; i++) { + ret = mtk_btag_pr_time(buf, len, name[i], t[i]); + buf += ret; + len -= ret; + } + + if (stage == max-1) { + busy_time = t[stage] - t[0]; + ret = mtk_btag_pr_time(buf, len, "busy", busy_time); + buf += ret; + len -= ret; + mtk_btag_pr_speed(buf, len, busy_time, bytes); + } +} +EXPORT_SYMBOL_GPL(mtk_btag_task_timetag); + + +void mtk_btag_seq_time(char **buff, unsigned long *size, + struct seq_file *seq, uint64_t time) +{ + uint32_t nsec; + + nsec = do_div(time, 1000000000); + SPREAD_PRINTF(buff, size, seq, "[%5lu.%06lu]", (unsigned long)time, + (unsigned long)nsec/1000); +} +EXPORT_SYMBOL_GPL(mtk_btag_seq_time); + +static void mtk_btag_seq_trace(char **buff, unsigned long *size, + struct seq_file *seq, const char *name, struct mtk_btag_trace *tr) +{ + int i; + + if (tr->time <= 0) + return; + + mtk_btag_seq_time(buff, size, seq, tr->time); + SPREAD_PRINTF(buff, size, seq, "%s.q:%d.", name, tr->qid); + + if (tr->throughput.r.usage) + SPREAD_PRINTF(buff, size, seq, biolog_fmt_rt, + tr->throughput.r.speed, + tr->throughput.r.size, + tr->throughput.r.usage); + if (tr->throughput.w.usage) + SPREAD_PRINTF(buff, size, seq, biolog_fmt_wt, + tr->throughput.w.speed, + tr->throughput.w.size, + tr->throughput.w.usage); + + SPREAD_PRINTF(buff, size, seq, biolog_fmt, + tr->workload.percent, + tr->workload.usage, + tr->workload.period, + tr->workload.count, + tr->vmstat.file_pages, + tr->vmstat.file_dirty, + tr->vmstat.dirtied, + tr->vmstat.writeback, + tr->vmstat.written, + tr->vmstat.fmflt, + tr->cpu.user, + tr->cpu.nice, + tr->cpu.system, + tr->cpu.idle, + tr->cpu.iowait, + tr->cpu.irq, + tr->cpu.softirq, + tr->pid); + + for (i = 0; i < BLOCKTAG_PIDLOG_ENTRIES; i++) { + struct mtk_btag_pidlogger_entry *pe; + + pe = &tr->pidlog.info[i]; + + if (pe->pid == 0) + break; + + SPREAD_PRINTF(buff, size, seq, pidlog_fmt, + pe->pid, + pe->w.count, + pe->w.length, + pe->r.count, + pe->r.length); + } + SPREAD_PRINTF(buff, size, seq, ".\n"); +} + +/* get current trace in debugfs ring buffer */ +struct mtk_btag_trace *mtk_btag_curr_trace(struct mtk_btag_ringtrace *rt) +{ + if (rt) + return &rt->trace[rt->index]; + else + return NULL; +} +EXPORT_SYMBOL_GPL(mtk_btag_curr_trace); + +/* step to next trace in debugfs ring buffer */ +struct mtk_btag_trace *mtk_btag_next_trace(struct mtk_btag_ringtrace *rt) +{ + rt->index++; + if (rt->index >= rt->max) + rt->index = 0; + + return mtk_btag_curr_trace(rt); +} +EXPORT_SYMBOL_GPL(mtk_btag_next_trace); + +/* clear debugfs ring buffer */ +static void mtk_btag_clear_trace(struct mtk_btag_ringtrace *rt) +{ + unsigned long flags; + + spin_lock_irqsave(&rt->lock, flags); + memset(rt->trace, 0, (sizeof(struct mtk_btag_trace) * rt->max)); + rt->index = 0; + spin_unlock_irqrestore(&rt->lock, flags); +} + +static void mtk_btag_seq_debug_show_ringtrace(char **buff, unsigned long *size, + struct seq_file *seq, struct mtk_blocktag *btag) +{ + struct mtk_btag_ringtrace *rt = BTAG_RT(btag); + unsigned long flags; + int i, end; + + if (!rt) + return; + + if (rt->index >= rt->max || rt->index < 0) + rt->index = 0; + + SPREAD_PRINTF(buff, size, seq, "<%s: blocktag trace %s>\n", + btag->name, BLOCKIO_MIN_VER); + + spin_lock_irqsave(&rt->lock, flags); + end = (rt->index > 0) ? rt->index-1 : rt->max-1; + for (i = rt->index;;) { + mtk_btag_seq_trace(buff, size, seq, btag->name, &rt->trace[i]); + if (i == end) + break; + i = (i >= rt->max-1) ? 0 : i+1; + }; + spin_unlock_irqrestore(&rt->lock, flags); +} + + +static size_t mtk_btag_seq_sub_show_usedmem(char **buff, unsigned long *size, + struct seq_file *seq, struct mtk_blocktag *btag) +{ + size_t used_mem = 0; + size_t size_l; + + SPREAD_PRINTF(buff, size, seq, "<%s: memory usage>\n", btag->name); + SPREAD_PRINTF(buff, size, seq, "%s blocktag: %zu bytes\n", btag->name, + sizeof(struct mtk_blocktag)); + used_mem += sizeof(struct mtk_blocktag); + + if (BTAG_RT(btag)) { + size_l = (sizeof(struct mtk_btag_trace) * BTAG_RT(btag)->max); + SPREAD_PRINTF(buff, size, seq, + "%s debug ring buffer: %d traces * %zu = %zu bytes\n", + btag->name, + BTAG_RT(btag)->max, + sizeof(struct mtk_btag_trace), + size_l); + used_mem += size_l; + } + + if (BTAG_CTX(btag)) { + size_l = btag->ctx.size * btag->ctx.count; + SPREAD_PRINTF(buff, size, seq, + "%s queue context: %d contexts * %d = %zu bytes\n", + btag->name, + btag->ctx.count, + btag->ctx.size, + size_l); + used_mem += size_l; + } + + SPREAD_PRINTF(buff, size, seq, "%s aee buffer: %d bytes\n", btag->name, + BLOCKIO_AEE_BUFFER_SIZE); + used_mem += BLOCKIO_AEE_BUFFER_SIZE; + + SPREAD_PRINTF(buff, size, seq, + "%s sub-total: %zu KB\n", btag->name, used_mem >> 10); + return used_mem; +} + +/* clear all ringtraces */ +static ssize_t mtk_btag_main_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct mtk_blocktag *btag, *n; + + mutex_lock(&mtk_btag_list_lock); + list_for_each_entry_safe(btag, n, &mtk_btag_list, list) + mtk_btag_clear_trace(&btag->rt); + mutex_unlock(&mtk_btag_list_lock); + return count; +} + +/* clear ringtrace */ +static ssize_t mtk_btag_sub_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct seq_file *seq = file->private_data; + struct mtk_blocktag *btag; + + if (seq && seq->private) { + btag = seq->private; + mtk_btag_clear_trace(&btag->rt); + } + return count; +} + +/* seq file operations */ +static void *mtk_btag_seq_debug_start(struct seq_file *seq, loff_t *pos) +{ + unsigned int idx; + + if (*pos < 0 || *pos >= 1) + return NULL; + + idx = *pos + 1; + return (void *) ((unsigned long) idx); +} + +static void *mtk_btag_seq_debug_next(struct seq_file *seq, void *v, loff_t *pos) +{ + unsigned int idx; + + ++*pos; + if (*pos < 0 || *pos >= 1) + return NULL; + + idx = *pos + 1; + return (void *) ((unsigned long) idx); +} + +static void mtk_btag_seq_debug_stop(struct seq_file *seq, void *v) +{ +} + +static int mtk_btag_seq_sub_show(struct seq_file *seq, void *v) +{ + struct mtk_blocktag *btag = seq->private; + + if (btag) { + mtk_btag_seq_debug_show_ringtrace(NULL, NULL, seq, btag); + if (btag->seq_show) { + seq_printf(seq, "<%s: context info>\n", btag->name); + btag->seq_show(NULL, NULL, seq); + } + mtk_btag_seq_sub_show_usedmem(NULL, NULL, seq, btag); + } + return 0; +} + +static const struct seq_operations mtk_btag_seq_sub_ops = { + .start = mtk_btag_seq_debug_start, + .next = mtk_btag_seq_debug_next, + .stop = mtk_btag_seq_debug_stop, + .show = mtk_btag_seq_sub_show, +}; + +static int mtk_btag_sub_open(struct inode *inode, struct file *file) +{ + int rc; + + rc = seq_open(file, &mtk_btag_seq_sub_ops); + + if (!rc) { + struct seq_file *m = file->private_data; + struct dentry *entry = container_of(inode->i_dentry.first, + struct dentry, d_u.d_alias); + + if (entry && entry->d_parent) { + pr_notice("[BLOCK_TAG] %s: %s/%s\n", __func__, + entry->d_parent->d_name.name, + entry->d_name.name); + m->private = + mtk_btag_find_locked(entry->d_parent->d_name.name); + } + } + return rc; +} + +static const struct file_operations mtk_btag_sub_fops = { + .owner = THIS_MODULE, + .open = mtk_btag_sub_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, + .write = mtk_btag_sub_write, +}; + +static ssize_t mtk_btag_mictx_sub_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + int ret; + char cmd[MICTX_PROC_CMD_BUF_SIZE]; + + if (count == 0) + goto err; + else if (count > MICTX_PROC_CMD_BUF_SIZE) + count = MICTX_PROC_CMD_BUF_SIZE; + + ret = copy_from_user(cmd, ubuf, count); + + if (ret < 0) + goto err; + + if (cmd[0] == '1') + mtk_btag_mictx_enable(1); + else if (cmd[0] == '2') + mtk_btag_mictx_enable(0); + else if (cmd[0] == '3') + mtk_btag_mictx_debug = 1; + else if (cmd[0] == '4') + mtk_btag_mictx_debug = 0; + else { + pr_info("[pidmap] invalid arg: 0x%x\n", cmd[0]); + goto err; + } + + return count; + +err: + return -1; +} +static int mtk_btag_mctx_sub_show(struct seq_file *s, void *data) +{ + seq_puts(s, "\n"); + seq_puts(s, "Status:\n"); + seq_printf(s, " Ready: %d\n", mtk_btag_mictx_ready); + seq_printf(s, " Mictx Instance: %p\n", mtk_btag_mictx); + seq_puts(s, "Commands:\n"); + seq_puts(s, " Enable Mini Context : echo 1 > blocktag_mictx\n"); + seq_puts(s, " Disable Mini Context: echo 2 > blocktag_mictx\n"); + seq_puts(s, " Enable Self-Test : echo 3 > blocktag_mictx\n"); + seq_puts(s, " Disable Self-Test : echo 4 > blocktag_mictx\n"); + return 0; +} + +static int mtk_btag_mictx_sub_open(struct inode *inode, struct file *file) +{ + return single_open(file, mtk_btag_mctx_sub_show, inode->i_private); +} + +static const struct file_operations mtk_btag_mictx_sub_fops = { + .owner = THIS_MODULE, + .open = mtk_btag_mictx_sub_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, + .write = mtk_btag_mictx_sub_write, +}; + +struct mtk_blocktag *mtk_btag_alloc(const char *name, + unsigned int ringtrace_count, size_t ctx_size, unsigned int ctx_count, + mtk_btag_seq_f seq_show) +{ + struct mtk_blocktag *btag; + + if (!name || !ringtrace_count || !ctx_size || !ctx_count) + return NULL; + + btag = mtk_btag_find(name); + if (btag) { + pr_notice("[BLOCK_TAG] %s: blocktag %s already exists.\n", + __func__, name); + return NULL; + } + + btag = kmalloc(sizeof(struct mtk_blocktag), GFP_NOFS); + if (!btag) + return NULL; + + memset(btag, 0, sizeof(struct mtk_blocktag)); + btag->seq_show = seq_show; + btag->used_mem = sizeof(struct mtk_blocktag) + + (sizeof(struct mtk_btag_trace) * ringtrace_count) + + (ctx_count * ctx_size); + + /* ringtrace */ + btag->rt.index = 0; + btag->rt.max = ringtrace_count; + spin_lock_init(&btag->rt.lock); + btag->rt.trace = kmalloc_array(ringtrace_count, + sizeof(struct mtk_btag_trace), GFP_NOFS); + if (!btag->rt.trace) { + kfree(btag); + return NULL; + } + memset(btag->rt.trace, 0, + (sizeof(struct mtk_btag_trace) * ringtrace_count)); + strncpy(btag->name, name, BLOCKTAG_NAME_LEN-1); + spin_lock_init(&btag->rt.lock); + + /* context */ + btag->ctx.count = ctx_count; + btag->ctx.size = ctx_size; + btag->ctx.priv = kmalloc_array(ctx_count, ctx_size, GFP_NOFS); + if (!btag->ctx.priv) { + kfree(btag->rt.trace); + kfree(btag); + return NULL; + } + memset(btag->ctx.priv, 0, ctx_size * ctx_count); + + /* debugfs dentries */ + mtk_btag_init_debugfs(); + btag->dentry.droot = debugfs_create_dir(name, mtk_btag_droot); + + if (IS_ERR(btag->dentry.droot)) + goto out; + + btag->dentry.dmem = debugfs_create_u32("used_mem", 0440, + btag->dentry.droot, &btag->used_mem); + + if (IS_ERR(btag->dentry.dmem)) + goto out; + + btag->dentry.dklog = debugfs_create_u32("klog_enable", 0660, + btag->dentry.droot, &btag->klog_enable); + + if (IS_ERR(btag->dentry.dklog)) + goto out; + + btag->dentry.dlog = debugfs_create_file("blockio", S_IFREG | 0444, + btag->dentry.droot, (void *)0, &mtk_btag_sub_fops); + + if (IS_ERR(btag->dentry.dlog)) + goto out; + + btag->dentry.dlog_mictx = debugfs_create_file("blockio_mictx", + S_IFREG | 0444, + btag->dentry.droot, (void *)0, &mtk_btag_mictx_sub_fops); + + if (IS_ERR(btag->dentry.dlog_mictx)) + goto out; + +out: + spin_lock_init(&btag->prbuf.lock); + list_add(&btag->list, &mtk_btag_list); + + return btag; +} +EXPORT_SYMBOL_GPL(mtk_btag_alloc); + +void mtk_btag_free(struct mtk_blocktag *btag) +{ + if (!btag) + return; + + list_del(&btag->list); + debugfs_remove_recursive(btag->dentry.droot); + kfree(btag->ctx.priv); + kfree(btag->rt.trace); + kfree(btag); +} +EXPORT_SYMBOL_GPL(mtk_btag_free); + +static int __init mtk_btag_early_memory_info(void) +{ + phys_addr_t start, end; + + start = memblock_start_of_DRAM(); + end = memblock_end_of_DRAM(); + mtk_btag_system_dram_size = (unsigned long long)(end - start); + pr_debug("[BLOCK_TAG] DRAM: %pa - %pa, size: 0x%llx\n", &start, + &end, (unsigned long long)(end - start)); + return 0; +} +fs_initcall(mtk_btag_early_memory_info); + +static void mtk_btag_pidlogger_init(void) +{ + unsigned long count = mtk_btag_system_dram_size >> PAGE_SHIFT; + unsigned long size = count * sizeof(struct page_pid_logger); + + if (mtk_btag_pagelogger) + goto init; + +#ifdef CONFIG_MTK_USE_RESERVED_EXT_MEM + mtk_btag_pagelogger = extmem_malloc_page_align(size); +#else + mtk_btag_pagelogger = vmalloc(size); +#endif + +init: + if (mtk_btag_pagelogger) + memset(mtk_btag_pagelogger, -1, size); + else + pr_info( + "[BLOCK_TAG] blockio: fail to allocate mtk_btag_pagelogger\n"); +} + +static void mtk_btag_seq_main_info(char **buff, unsigned long *size, + struct seq_file *seq) +{ + size_t used_mem = 0; + struct mtk_blocktag *btag, *n; + + SPREAD_PRINTF(buff, size, seq, "[Trace]\n"); + mutex_lock(&mtk_btag_list_lock); + list_for_each_entry_safe(btag, n, &mtk_btag_list, list) + mtk_btag_seq_debug_show_ringtrace(buff, size, seq, btag); + + SPREAD_PRINTF(buff, size, seq, "[Info]\n"); + list_for_each_entry_safe(btag, n, &mtk_btag_list, list) + if (btag->seq_show) { + SPREAD_PRINTF(buff, size, seq, "<%s: context info>\n", + btag->name); + btag->seq_show(buff, size, seq); + } + + SPREAD_PRINTF(buff, size, seq, "[Memory Usage]\n"); + list_for_each_entry_safe(btag, n, &mtk_btag_list, list) + used_mem += mtk_btag_seq_sub_show_usedmem(buff, size, + seq, btag); + mutex_unlock(&mtk_btag_list_lock); + + SPREAD_PRINTF(buff, size, seq, "\n"); + used_mem += mtk_btag_seq_pidlog_usedmem(buff, size, seq); + + SPREAD_PRINTF(buff, size, seq, "--------------------------------\n"); + SPREAD_PRINTF(buff, size, seq, "Total: %zu KB\n", used_mem >> 10); +} + +static int mtk_btag_seq_main_show(struct seq_file *seq, void *v) +{ + mtk_btag_seq_main_info(NULL, NULL, seq); + return 0; +} + +void get_blockio_aee_buffer(unsigned long *vaddr, unsigned long *size) +{ + unsigned long free_size = BLOCKIO_AEE_BUFFER_SIZE; + char *buff; + + buff = blockio_aee_buffer; + mtk_btag_seq_main_info(&buff, &free_size, NULL); + /* retrun start location */ + *vaddr = (unsigned long)blockio_aee_buffer; + *size = BLOCKIO_AEE_BUFFER_SIZE - free_size; +} +EXPORT_SYMBOL(get_blockio_aee_buffer); + +static const struct seq_operations mtk_btag_seq_main_ops = { + .start = mtk_btag_seq_debug_start, + .next = mtk_btag_seq_debug_next, + .stop = mtk_btag_seq_debug_stop, + .show = mtk_btag_seq_main_show, +}; + +static int mtk_btag_main_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &mtk_btag_seq_main_ops); +} + +static const struct file_operations mtk_btag_main_fops = { + .owner = THIS_MODULE, + .open = mtk_btag_main_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, + .write = mtk_btag_main_write, +}; + +static void mtk_btag_init_debugfs(void) +{ + if (mtk_btag_droot) + return; + + mtk_btag_droot = debugfs_create_dir("blocktag", NULL); + if (IS_ERR(mtk_btag_droot)) { + mtk_btag_droot = NULL; + return; + } + + mtk_btag_dlog = debugfs_create_file("blockio", S_IFREG | 0444, NULL, + (void *)0, &mtk_btag_main_fops); +} + +static inline +struct mtk_btag_mictx_struct *mtk_btag_mictx_get_ctx(void) +{ + if (mtk_btag_mictx_ready) + return mtk_btag_mictx; + else + return NULL; +} + +void mtk_btag_mictx_eval_tp( + unsigned int write, __u64 usage, __u32 size) +{ + struct mtk_btag_mictx_struct *ctx; + struct mtk_btag_throughput_rw *tprw; + unsigned long flags; + __u64 cur_time = sched_clock(); + __u64 req_begin_time; + + ctx = mtk_btag_mictx_get_ctx(); + if (!ctx) + return; + + tprw = (write) ? &ctx->tp.w : &ctx->tp.r; + spin_lock_irqsave(&ctx->lock, flags); + tprw->size += size; + tprw->usage += usage; + + ctx->tp_max_time = cur_time; + req_begin_time = cur_time - usage; + + if (!ctx->tp_min_time) + ctx->tp_min_time = req_begin_time; + else { + if (req_begin_time < ctx->tp_min_time) + ctx->tp_min_time = req_begin_time; + } + spin_unlock_irqrestore(&ctx->lock, flags); +} + +void mtk_btag_mictx_eval_req( + unsigned int write, __u32 cnt, __u32 size) +{ + struct mtk_btag_mictx_struct *ctx; + struct mtk_btag_req_rw *reqrw; + unsigned long flags; + + ctx = mtk_btag_mictx_get_ctx(); + if (!ctx) + return; + + reqrw = (write) ? &ctx->req.w : &ctx->req.r; + spin_lock_irqsave(&ctx->lock, flags); + reqrw->count += cnt; + reqrw->size += size; + spin_unlock_irqrestore(&ctx->lock, flags); +} + +void mtk_btag_mictx_update_ctx( + __u32 q_depth) +{ + struct mtk_btag_mictx_struct *ctx; + unsigned long flags; + + ctx = mtk_btag_mictx_get_ctx(); + if (!ctx) + return; + + spin_lock_irqsave(&ctx->lock, flags); + ctx->q_depth = q_depth; + + if (!ctx->q_depth) { + ctx->idle_begin = sched_clock(); + } else { + if (ctx->idle_begin) { + ctx->idle_total += + (sched_clock() - ctx->idle_begin); + ctx->idle_begin = 0; + } + } + spin_unlock_irqrestore(&ctx->lock, flags); +} + +static void mtk_btag_mictx_reset( + struct mtk_btag_mictx_struct *ctx, + __u64 window_begin) +{ + if (!window_begin) + window_begin = sched_clock(); + ctx->window_begin = window_begin; + + if (!ctx->q_depth) + ctx->idle_begin = ctx->window_begin; + else + ctx->idle_begin = 0; + + ctx->idle_total = 0; + ctx->tp_min_time = ctx->tp_max_time = 0; + memset(&ctx->tp, 0, sizeof(struct mtk_btag_throughput)); + memset(&ctx->req, 0, sizeof(struct mtk_btag_req)); +} + +int mtk_btag_mictx_get_data( + struct mtk_btag_mictx_iostat_struct *iostat) +{ + struct mtk_btag_mictx_struct *ctx; + __u64 time_cur, dur, tp_dur; + unsigned long flags; + + ctx = mtk_btag_mictx_get_ctx(); + if (!ctx || !iostat) + return -1; + + spin_lock_irqsave(&ctx->lock, flags); + + time_cur = sched_clock(); + dur = time_cur - ctx->window_begin; + + /* fill-in duration */ + iostat->duration = dur; + + /* calculate throughput (per-request) */ + iostat->tp_req_r = mtk_btag_eval_tp_speed( + ctx->tp.r.size, ctx->tp.r.usage); + iostat->tp_req_w = mtk_btag_eval_tp_speed( + ctx->tp.w.size, ctx->tp.w.usage); + + /* calculate throughput (overlapped, not 100% precise) */ + tp_dur = ctx->tp_max_time - ctx->tp_min_time; + iostat->tp_all_r = mtk_btag_eval_tp_speed( + ctx->tp.r.size, tp_dur); + iostat->tp_all_w = mtk_btag_eval_tp_speed( + ctx->tp.w.size, tp_dur); + + /* provide request count and size */ + iostat->reqcnt_r = ctx->req.r.count; + iostat->reqsize_r = ctx->req.r.size; + iostat->reqcnt_w = ctx->req.w.count; + iostat->reqsize_w = ctx->req.w.size; + + /* calculate workload */ + if (ctx->idle_begin) + ctx->idle_total += (time_cur - ctx->idle_begin); + + iostat->wl = 100 - + ((__u32)((ctx->idle_total >> 10) * 100) / (__u32)(dur >> 10)); + + /* fill-in cmdq depth */ + iostat->q_depth = ctx->q_depth; + + /* everything was provided, now we can reset the ctx */ + mtk_btag_mictx_reset(ctx, time_cur); + + spin_unlock_irqrestore(&ctx->lock, flags); + + return 0; +} + +void mtk_btag_mictx_enable(int enable) +{ + if (enable && mtk_btag_mictx) + return; + + if (enable) { + mtk_btag_mictx = + kzalloc(sizeof(struct mtk_btag_mictx_struct), GFP_NOFS); + if (!mtk_btag_mictx) { + pr_info("[BLOCK_TAG] mtk_btag_mictx allocation fail, disabled.\n"); + return; + } + + spin_lock_init(&mtk_btag_mictx->lock); + mtk_btag_mictx_reset(mtk_btag_mictx, 0); + mtk_btag_mictx_ready = 1; + + } else { + if (!mtk_btag_mictx) + return; + + mtk_btag_mictx_ready = 0; + kfree(mtk_btag_mictx); + mtk_btag_mictx = NULL; + } +} + +static int __init mtk_btag_init(void) +{ + mtk_btag_pidlogger_init(); + mtk_btag_init_debugfs(); + return 0; +} + +static void __exit mtk_btag_exit(void) +{ + debugfs_remove(mtk_btag_dlog); +} + +module_init(mtk_btag_init); +module_exit(mtk_btag_exit); + +MODULE_AUTHOR("Perry Hsu "); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Storage Block Tag Trace"); + diff --git a/drivers/misc/mediatek/include/mt-plat/mtk_blocktag.h b/drivers/misc/mediatek/include/mt-plat/mtk_blocktag.h new file mode 100644 index 000000000000..9387b5a54025 --- /dev/null +++ b/drivers/misc/mediatek/include/mt-plat/mtk_blocktag.h @@ -0,0 +1,262 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 MediaTek Inc. + */ + +#ifndef _MTK_BLOCKTAG_H +#define _MTK_BLOCKTAG_H + +#include +#include + +#if defined(CONFIG_MTK_BLOCK_TAG) + +/* + * MTK_BTAG_FEATURE_MICTX_IOSTAT + * + * Shall be defined if we can provide iostat + * produced by mini context. + * + * This feature is used to extend kernel + * trace events to have more I/O information. + */ +#define MTK_BTAG_FEATURE_MICTX_IOSTAT + +#define BLOCKTAG_PIDLOG_ENTRIES 50 +#define BLOCKTAG_NAME_LEN 16 +#define BLOCKTAG_PRINT_LEN 4096 + +#define BTAG_RT(btag) (btag ? &btag->rt : NULL) +#define BTAG_CTX(btag) (btag ? btag->ctx.priv : NULL) +#define BTAG_KLOGEN(btag) (btag ? btag->klog_enable : 0) + +struct page_pid_logger { + unsigned short pid; +}; + +#ifdef CONFIG_MTK_USE_RESERVED_EXT_MEM +extern void *extmem_malloc_page_align(size_t bytes); +#endif + +enum { + PIDLOG_MODE_BLK_SUBMIT_BIO = 0, + PIDLOG_MODE_MM_FS +}; + +enum mtk_btag_storage_type { + BTAG_STORAGE_EMBEDDED = 0, + BTAG_STORAGE_EXTERNAL +}; + +struct mtk_btag_workload { + __u64 period; /* period time (ns) */ + __u64 usage; /* busy time (ns) */ + __u32 percent; /* workload */ + __u32 count; /* access count */ +}; + +struct mtk_btag_throughput_rw { + __u64 usage; /* busy time (ns) */ + __u32 size; /* transferred bytes */ + __u32 speed; /* KB/s */ +}; + +struct mtk_btag_throughput { + struct mtk_btag_throughput_rw r; /* read */ + struct mtk_btag_throughput_rw w; /* write */ +}; + +struct mtk_btag_req_rw { + __u16 count; + __u32 size; /* bytes */ +}; + +struct mtk_btag_req { + struct mtk_btag_req_rw r; /* read */ + struct mtk_btag_req_rw w; /* write */ +}; + +/* + * public structure to provide IO statistics + * in a period of time. + * + * Make sure MTK_BTAG_FEATURE_MICTX_IOSTAT is + * defined alone with mictx series. + */ +struct mtk_btag_mictx_iostat_struct { + __u64 duration; /* duration time for below performance data (ns) */ + __u32 tp_req_r; /* throughput (per-request): read (KB/s) */ + __u32 tp_req_w; /* throughput (per-request): write (KB/s) */ + __u32 tp_all_r; /* throughput (overlapped) : read (KB/s) */ + __u32 tp_all_w; /* throughput (overlapped) : write (KB/s) */ + __u32 reqsize_r; /* request size : read (Bytes) */ + __u32 reqsize_w; /* request size : write (Bytes) */ + __u32 reqcnt_r; /* request count: read */ + __u32 reqcnt_w; /* request count: write */ + __u16 wl; /* storage device workload (%) */ + __u16 q_depth; /* storage cmdq queue depth */ +}; + +/* + * mini context for integration with + * other performance analysis tools. + */ +struct mtk_btag_mictx_struct { + struct mtk_btag_throughput tp; + struct mtk_btag_req req; + __u64 window_begin; + __u64 tp_min_time; + __u64 tp_max_time; + __u64 idle_begin; + __u64 idle_total; + __u32 q_depth; + spinlock_t lock; +}; + +struct mtk_btag_vmstat { + __u64 file_pages; + __u64 file_dirty; + __u64 dirtied; + __u64 writeback; + __u64 written; + __u64 fmflt; +}; + +struct mtk_btag_pidlogger_entry_rw { + __u16 count; + __u32 length; +}; + +struct mtk_btag_pidlogger_entry { + __u16 pid; + struct mtk_btag_pidlogger_entry_rw r; /* read */ + struct mtk_btag_pidlogger_entry_rw w; /* write */ +}; + +struct mtk_btag_pidlogger { + __u16 current_pid; + struct mtk_btag_pidlogger_entry info[BLOCKTAG_PIDLOG_ENTRIES]; +}; + +struct mtk_btag_cpu { + __u64 user; + __u64 nice; + __u64 system; + __u64 idle; + __u64 iowait; + __u64 irq; + __u64 softirq; +}; + +/* Trace: entry of the ring buffer */ +struct mtk_btag_trace { + uint64_t time; + pid_t pid; + u32 qid; + struct mtk_btag_workload workload; + struct mtk_btag_throughput throughput; + struct mtk_btag_vmstat vmstat; + struct mtk_btag_pidlogger pidlog; + struct mtk_btag_cpu cpu; +}; + +/* Ring Trace */ +struct mtk_btag_ringtrace { + struct mtk_btag_trace *trace; + spinlock_t lock; + int index; + int max; +}; + +typedef size_t (*mtk_btag_seq_f) (char **, unsigned long *, struct seq_file *); + +/* BlockTag */ +struct mtk_blocktag { + char name[BLOCKTAG_NAME_LEN]; + struct mtk_btag_ringtrace rt; + + struct prbuf_t { + spinlock_t lock; + char buf[BLOCKTAG_PRINT_LEN]; + } prbuf; + + /* lock order: ctx.priv->lock => prbuf.lock */ + struct context_t { + int count; + int size; + void *priv; + } ctx; + + struct dentry_t { + struct dentry *droot; + struct dentry *dklog; + struct dentry *dlog; + struct dentry *dlog_mictx; + struct dentry *dmem; + } dentry; + + mtk_btag_seq_f seq_show; + + unsigned int klog_enable; + unsigned int used_mem; + + struct list_head list; +}; + +struct mtk_blocktag *mtk_btag_alloc(const char *name, + unsigned int ringtrace_count, size_t ctx_size, unsigned int ctx_count, + mtk_btag_seq_f seq_show); +void mtk_btag_free(struct mtk_blocktag *btag); + +struct mtk_btag_trace *mtk_btag_curr_trace(struct mtk_btag_ringtrace *rt); +struct mtk_btag_trace *mtk_btag_next_trace(struct mtk_btag_ringtrace *rt); + +int mtk_btag_pidlog_add_mmc(struct request_queue *q, pid_t pid, __u32 len, + int rw); +int mtk_btag_pidlog_add_ufs(struct request_queue *q, pid_t pid, __u32 len, + int rw); +void mtk_btag_pidlog_insert(struct mtk_btag_pidlogger *pidlog, pid_t pid, +__u32 len, int rw); + +void mtk_btag_cpu_eval(struct mtk_btag_cpu *cpu); +void mtk_btag_pidlog_eval(struct mtk_btag_pidlogger *pl, + struct mtk_btag_pidlogger *ctx_pl); +void mtk_btag_throughput_eval(struct mtk_btag_throughput *tp); +void mtk_btag_vmstat_eval(struct mtk_btag_vmstat *vm); + +void mtk_btag_task_timetag(char *buf, unsigned int len, unsigned int stage, + unsigned int max, const char *name[], uint64_t *t, __u32 bytes); +void mtk_btag_klog(struct mtk_blocktag *btag, struct mtk_btag_trace *tr); + +void mtk_btag_pidlog_map_sg(struct request_queue *q, struct bio *bio, + struct bio_vec *bvec); +void mtk_btag_pidlog_copy_pid(struct page *src, struct page *dst); +void mtk_btag_pidlog_submit_bio(struct bio *bio); +void mtk_btag_pidlog_set_pid(struct page *p); + +void mtk_btag_mictx_enable(int enable); +void mtk_btag_mictx_eval_tp( + unsigned int rw, __u64 usage, __u32 size); +void mtk_btag_mictx_eval_req( + unsigned int rw, __u32 cnt, __u32 size); +int mtk_btag_mictx_get_data( + struct mtk_btag_mictx_iostat_struct *iostat); +void mtk_btag_mictx_update_ctx(__u32 q_depth); + +#else + +#define mtk_btag_pidlog_copy_pid(...) +#define mtk_btag_pidlog_map_sg(...) +#define mtk_btag_pidlog_submit_bio(...) +#define mtk_btag_pidlog_set_pid(...) + +#define mtk_btag_mictx_enable(...) +#define mtk_btag_mictx_eval_tp(...) +#define mtk_btag_mictx_eval_req(...) +#define mtk_btag_mictx_get_data(...) +#define mtk_btag_mictx_update_ctx(...) + +#endif + +#endif + diff --git a/drivers/mmc/core/Kconfig b/drivers/mmc/core/Kconfig index 3ac18c536220..10a90c220285 100644 --- a/drivers/mmc/core/Kconfig +++ b/drivers/mmc/core/Kconfig @@ -89,3 +89,16 @@ config MMC_CRYPTO capabilities of the MMC device (if present) to perform crypto operations on data being transferred to/from the device. +config MMC_BLOCK_IO_LOG + bool "Block IO log" + depends on MMC_BLOCK + depends on MTK_BLOCK_TAG + default y + help + The BLOCK TAG trace provides I/O logs for performance analysis and + application access pattern study. The BLOCK TAG trace summarizes the + I/O of each process every 1 second, and saves to ring buffer. The log + can be displayed by "cat /sys/kernel/debug/blockio". + The trace is default enabled on user and eng builts, say N here to + disable it. + diff --git a/drivers/mmc/core/Makefile b/drivers/mmc/core/Makefile index d4658142ed16..7cd3f67a88b0 100644 --- a/drivers/mmc/core/Makefile +++ b/drivers/mmc/core/Makefile @@ -19,3 +19,4 @@ mmc_block-objs := block.o queue.o obj-$(CONFIG_MMC_TEST) += mmc_test.o obj-$(CONFIG_SDIO_UART) += sdio_uart.o obj-$(CONFIG_MMC_CRYPTO) += mmc-crypto.o +obj-$(CONFIG_MMC_BLOCK_IO_LOG) += mtk_mmc_block.o diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 29f4781db785..2b3099f13fae 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -47,6 +47,7 @@ #include +#include "mtk_mmc_block.h" #include "queue.h" #include "block.h" #include "core.h" @@ -2230,12 +2231,14 @@ static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq, err = mmc_blk_rw_wait(mq, &prev_req); if (err) goto out_post_req; + mt_biolog_mmcqd_req_end(mqrq->brq.mrq.data); mq->rw_wait = true; err = mmc_start_request(host, &mqrq->brq.mrq); if (prev_req) mmc_blk_mq_post_req(mq, prev_req); + mt_biolog_mmcqd_req_start(host); if (err) mq->rw_wait = false; @@ -2308,6 +2311,7 @@ static int mmc_blk_swcq_issue_rw_rq(struct mmc_queue *mq, else return -EBUSY; + mt_biolog_mmcqd_req_check(); mq->mqrq[index].req = req; atomic_set(&mqrq->index, index + 1); atomic_set(&mq->mqrq[index].index, index + 1); @@ -3212,7 +3216,7 @@ static int __init mmc_blk_init(void) res = mmc_register_driver(&mmc_driver); if (res) goto out_blkdev_unreg; - + mt_mmc_biolog_init(); return 0; out_blkdev_unreg: diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index f7d87d70f452..a8b0fb0c4bc0 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -51,6 +51,7 @@ #include "mmc_ops.h" #include "sd_ops.h" #include "sdio_ops.h" +#include "mtk_mmc_block.h" /* The max erase timeout, used when host->max_busy_timeout isn't specified */ #define MMC_ERASE_TIMEOUT_MS (60 * 1000) /* 60 s */ @@ -897,9 +898,10 @@ int mmc_run_queue_thread(void *data) u64 chk_time = 0; pr_info("[CQ] start cmdq thread\n"); + mt_bio_queue_alloc(current, NULL); while (1) { - + mt_biolog_cmdq_check(); /* End request stage 1/2 */ if (atomic_read(&host->cq_rw) || (atomic_read(&host->areq_cnt) <= 1)) { @@ -942,7 +944,7 @@ int mmc_run_queue_thread(void *data) if (done_mrq && !done_mrq->data->error && !done_mrq->cmd->error) { task_id = (done_mrq->cmd->arg >> 16) & 0x1f; - //mt_biolog_cmdq_dma_end(task_id); + mt_biolog_cmdq_dma_end(task_id); mmc_check_write(host, done_mrq); host->cur_rw_task = CQ_TASK_IDLE; is_done = true; @@ -982,7 +984,7 @@ int mmc_run_queue_thread(void *data) if (err == -EINVAL) WARN_ON(1); host->ops->request(host, dat_mrq); - //mt_biolog_cmdq_dma_start(task_id); + mt_biolog_cmdq_dma_start(task_id); atomic_dec(&host->cq_rdy_cnt); dat_mrq = NULL; } @@ -991,10 +993,10 @@ int mmc_run_queue_thread(void *data) /* End request stage 2/2 */ if (is_done) { task_id = (done_mrq->cmd->arg >> 16) & 0x1f; - //mt_biolog_cmdq_isdone_start(task_id, - // host->areq_que[task_id]->mrq_que); - //mt_biolog_cmdq_isdone_end(task_id); - //mt_biolog_cmdq_check(); + mt_biolog_cmdq_isdone_start(task_id, + host->areq_que[task_id]->mrq_que); + mt_biolog_cmdq_isdone_end(task_id); + mt_biolog_cmdq_check(); mmc_blk_end_queued_req(host, done_mrq->areq, task_id); done_mrq = NULL; is_done = false; @@ -1009,7 +1011,7 @@ int mmc_run_queue_thread(void *data) while (cmd_mrq) { task_id = ((cmd_mrq->sbc->arg >> 16) & 0x1f); - //mt_biolog_cmdq_queue_task(task_id, cmd_mrq); + mt_biolog_cmdq_queue_task(task_id, cmd_mrq); if (host->task_id_index & (1 << task_id)) { pr_info( "[%s] BUG!!! task_id %d used, task_id_index 0x%08lx, areq_cnt = %d, cq_wait_rdy = %d\n", @@ -1089,14 +1091,14 @@ int mmc_run_queue_thread(void *data) } /* Sleep when nothing to do */ - //mt_biolog_cmdq_check(); + mt_biolog_cmdq_check(); set_current_state(TASK_INTERRUPTIBLE); if (atomic_read(&host->areq_cnt) == 0) schedule(); set_current_state(TASK_RUNNING); } - //mt_bio_queue_free(current); + mt_bio_queue_free(current); return 0; } #endif diff --git a/drivers/mmc/core/mtk_mmc_block.c b/drivers/mmc/core/mtk_mmc_block.c new file mode 100644 index 000000000000..6bfe9002f7ba --- /dev/null +++ b/drivers/mmc/core/mtk_mmc_block.c @@ -0,0 +1,915 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 MediaTek Inc. + */ + +#define DEBUG 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_MTK_USE_RESERVED_EXT_MEM +#include +#endif + +#include "mtk_mmc_block.h" +#include + +#define SECTOR_SHIFT 9 + +/* ring trace for debugfs */ +struct mtk_blocktag *mtk_btag_mmc; + +/* context buffer of each request queue */ +struct mt_bio_context *mt_ctx_map[MMC_BIOLOG_CONTEXTS] = { 0 }; + +/* context index for mt_ctx_map */ +enum { + CTX_MMCQD0 = 0, + CTX_MMCQD1 = 1, + CTX_MMCQD0_BOOT0 = 2, + CTX_MMCQD0_BOOT1 = 3, + CTX_MMCQD0_RPMB = 4, + CTX_MMCCMDQD0 = 5, + CTX_EXECQ = 9 +}; + +/* context state for command queue */ +enum { + CMDQ_CTX_NOT_DMA = 0, + CMDQ_CTX_IN_DMA = 1, + CMDQ_CTX_QUEUE = 2 +}; + +/* context state for mmcqd */ +enum { + MMCQD_NORMAL = 0, + MMCQD_CMDQ_MODE_EN = 1 +}; + +#define MT_BIO_TRACE_LATENCY (unsigned long long)(1000000000) + +#define REQ_EXECQ "exe_cq" +#define REQ_MMCQD0 "kworker" +#define REQ_MMCQD0_BOOT0 "mmcqd/0boot0" +#define REQ_MMCQD0_BOOT1 "mmcqd/0boot1" +#define REQ_MMCQD0_RPMB "mmcqd/0rpmb" +#define REQ_MMCCMDQD0 "mmc-cmdqd/0" +#define REQ_MMCQD1 "mmcqd/1" + +static void mt_bio_ctx_count_usage(struct mt_bio_context *ctx, + __u64 start, __u64 end); +static uint64_t mt_bio_get_period_busy(struct mt_bio_context *ctx); + +/* queue id: + * 0=internal storage (emmc:mmcqd0/exe_cq), + * 1=external storage (t-card:mmcqd1) + */ +static int get_qid_by_name(const char *str) +{ + if (strncmp(str, REQ_EXECQ, strlen(REQ_EXECQ)) == 0) + return BTAG_STORAGE_EMBEDDED; + if (strncmp(str, REQ_MMCQD0, strlen(REQ_MMCQD0)) == 0) + return BTAG_STORAGE_EMBEDDED; /* this includes boot0, boot1 */ + if (strncmp(str, REQ_MMCCMDQD0, strlen(REQ_MMCCMDQD0)) == 0) + return BTAG_STORAGE_EMBEDDED; + if (strncmp(str, REQ_MMCQD1, strlen(REQ_MMCQD1)) == 0) + return BTAG_STORAGE_EXTERNAL; + return 99; +} + +/* get context id to mt_ctx_map[] by name */ +static int get_ctxid_by_name(const char *str) +{ + if (strncmp(str, REQ_EXECQ, strlen(REQ_EXECQ)) == 0) + return CTX_EXECQ; + if (strncmp(str, REQ_MMCQD0_RPMB, strlen(REQ_MMCQD0_RPMB)) == 0) + return CTX_MMCQD0_RPMB; + if (strncmp(str, REQ_MMCQD0_BOOT0, strlen(REQ_MMCQD0_BOOT0)) == 0) + return CTX_MMCQD0_BOOT0; + if (strncmp(str, REQ_MMCQD0_BOOT1, strlen(REQ_MMCQD0_BOOT1)) == 0) + return CTX_MMCQD0_BOOT1; + if (strncmp(str, REQ_MMCCMDQD0, strlen(REQ_MMCCMDQD0)) == 0) + return CTX_MMCCMDQD0; + if (strncmp(str, REQ_MMCQD0, strlen(REQ_MMCQD0)) == 0) + return CTX_MMCQD0; + if (strncmp(str, REQ_MMCQD1, strlen(REQ_MMCQD1)) == 0) + return CTX_MMCQD1; + return -1; +} + +static void mt_bio_init_task(struct mt_bio_context_task *tsk) +{ + int i; + + tsk->task_id = -1; + tsk->arg = 0; + for (i = 0; i < tsk_max; i++) + tsk->t[i] = 0; +} + +static void mt_bio_init_ctx(struct mt_bio_context *ctx, + struct task_struct *thread, struct request_queue *q) +{ + int i; + + ctx->q = q; + ctx->pid = task_pid_nr(thread); + get_task_comm(ctx->comm, thread); + ctx->qid = get_qid_by_name(ctx->comm); + spin_lock_init(&ctx->lock); + ctx->id = get_ctxid_by_name(ctx->comm); + if (ctx->id >= 0) + mt_ctx_map[ctx->id] = ctx; + ctx->period_start_t = sched_clock(); + + for (i = 0; i < MMC_BIOLOG_CONTEXT_TASKS; i++) + mt_bio_init_task(&ctx->task[i]); +} + +void mt_bio_queue_alloc(struct task_struct *thread, struct request_queue *q) +{ + int i; + pid_t pid; + struct mt_bio_context *ctx = BTAG_CTX(mtk_btag_mmc); + + if (!ctx) + return; + + pid = task_pid_nr(thread); + + for (i = 0; i < MMC_BIOLOG_CONTEXTS; i++) { + if (ctx[i].pid == pid) + break; + if (ctx[i].pid == 0) { + mt_bio_init_ctx(&ctx[i], thread, q); + break; + } + } +} + +void mt_bio_queue_free(struct task_struct *thread) +{ + int i; + pid_t pid; + struct mt_bio_context *ctx = BTAG_CTX(mtk_btag_mmc); + + if (!ctx) + return; + + pid = task_pid_nr(thread); + + for (i = 0; i < MMC_BIOLOG_CONTEXTS; i++) { + if (ctx[i].pid == pid) { + mt_ctx_map[ctx[i].id] = NULL; + memset(&ctx[i], 0, sizeof(struct mt_bio_context)); + break; + } + } +} + +static struct mt_bio_context *mt_bio_curr_queue(struct request_queue *q) +{ + int i; + pid_t qd_pid; + struct mt_bio_context *ctx = BTAG_CTX(mtk_btag_mmc); + + if (!ctx) + return NULL; + + qd_pid = task_pid_nr(current); + + for (i = 0; i < MMC_BIOLOG_CONTEXTS; i++) { + if (ctx[i].pid == 0) + continue; + if ((qd_pid == ctx[i].pid) || (ctx[i].q && ctx[i].q == q)) + return &ctx[i]; + } + + return NULL; +} + +/* get context correspond to current process */ +static inline struct mt_bio_context *mt_bio_curr_ctx(void) +{ + return mt_bio_curr_queue(NULL); +} + +/* get other queue's context by context id */ +static struct mt_bio_context *mt_bio_get_ctx(int id) +{ + if (id < 0 || id >= MMC_BIOLOG_CONTEXTS) + return NULL; + + return mt_ctx_map[id]; +} + +/* append a pidlog to given context */ +int mtk_btag_pidlog_add_mmc(struct request_queue *q, pid_t pid, __u32 len, + int write) +{ + unsigned long flags; + struct mt_bio_context *ctx; + + ctx = mt_bio_curr_queue(q); + if (!ctx) + return 0; + + spin_lock_irqsave(&ctx->lock, flags); + mtk_btag_pidlog_insert(&ctx->pidlog, pid, len, write); + + if (ctx->qid == BTAG_STORAGE_EMBEDDED) + mtk_btag_mictx_eval_req(write, 1, len); + + spin_unlock_irqrestore(&ctx->lock, flags); + + return 1; +} +EXPORT_SYMBOL_GPL(mtk_btag_pidlog_add_mmc); + +/* evaluate throughput and workload of given context */ +static void mt_bio_context_eval(struct mt_bio_context *ctx) +{ + struct mt_bio_context_task *tsk; + uint64_t min, period, tsk_start; + int i; + + min = ctx->period_end_t; + + /* for all tasks if there is an on-going request */ + for (i = 0; i < MMC_BIOLOG_CONTEXT_TASKS; i++) { + tsk = &ctx->task[i]; + if (tsk->task_id >= 0) { + tsk_start = tsk->t[tsk_req_start]; + if (tsk_start && + tsk_start >= ctx->period_start_t && + tsk_start < min) + min = tsk_start; + } + } + + mt_bio_ctx_count_usage(ctx, min, ctx->period_end_t); + ctx->workload.usage = mt_bio_get_period_busy(ctx); + + if (ctx->workload.period > (ctx->workload.usage * 100)) { + ctx->workload.percent = 1; + } else { + period = ctx->workload.period; + do_div(period, 100); + ctx->workload.percent = + (__u32)ctx->workload.usage / (__u32)period; + } + + mtk_btag_throughput_eval(&ctx->throughput); +} + +/* print context to trace ring buffer */ +static void mt_bio_print_trace(struct mt_bio_context *ctx) +{ + struct mtk_btag_ringtrace *rt = BTAG_RT(mtk_btag_mmc); + struct mtk_btag_trace *tr; + struct mt_bio_context *pid_ctx = ctx; + unsigned long flags; + + if (!rt) + return; + + if (ctx->id == CTX_EXECQ) + pid_ctx = mt_bio_get_ctx(CTX_MMCQD0); + + spin_lock_irqsave(&rt->lock, flags); + tr = mtk_btag_curr_trace(rt); + + if (!tr) + goto out; + + memset(tr, 0, sizeof(struct mtk_btag_trace)); + tr->pid = ctx->pid; + tr->qid = ctx->qid; + memcpy(&tr->throughput, &ctx->throughput, + sizeof(struct mtk_btag_throughput)); + memcpy(&tr->workload, &ctx->workload, sizeof(struct mtk_btag_workload)); + + if (pid_ctx) + mtk_btag_pidlog_eval(&tr->pidlog, &pid_ctx->pidlog); + + mtk_btag_vmstat_eval(&tr->vmstat); + mtk_btag_cpu_eval(&tr->cpu); + + tr->time = sched_clock(); + + mtk_btag_klog(mtk_btag_mmc, tr); + mtk_btag_next_trace(rt); +out: + spin_unlock_irqrestore(&rt->lock, flags); +} + + +static struct mt_bio_context_task *mt_bio_get_task(struct mt_bio_context *ctx, + unsigned int task_id) +{ + struct mt_bio_context_task *tsk; + + if (!ctx) + return NULL; + + if (task_id >= MMC_BIOLOG_CONTEXT_TASKS) { + pr_notice("%s: invalid task id %d\n", + __func__, task_id); + return NULL; + } + + tsk = &ctx->task[task_id]; + tsk->task_id = task_id; + + return tsk; +} + +static struct mt_bio_context_task *mt_bio_curr_task_by_ctx_id( + unsigned int task_id, + struct mt_bio_context **curr_ctx, int mt_ctx_map) +{ + struct mt_bio_context *ctx; + + if (mt_ctx_map == -1) + /* get ctx by current pid */ + ctx = mt_bio_curr_ctx(); + else + /* get ctx by ctx map id */ + ctx = mt_bio_get_ctx(mt_ctx_map); + if (curr_ctx) + *curr_ctx = ctx; + return mt_bio_get_task(ctx, task_id); +} + +static struct mt_bio_context_task *mt_bio_curr_task(unsigned int task_id, + struct mt_bio_context **curr_ctx) +{ + /* get ctx by current pid */ + return mt_bio_curr_task_by_ctx_id(task_id, curr_ctx, -1); +} + +static const char *task_name[tsk_max] = { + "req_start", "dma_start", "dma_end", "isdone_start", "isdone_end"}; + +static void mt_pr_cmdq_tsk(struct mt_bio_context_task *tsk, int stage) +{ + int rw; + int klogen = BTAG_KLOGEN(mtk_btag_mmc); + __u32 bytes; + char buf[256]; + + if (!((klogen == 2 && stage == tsk_isdone_end) || (klogen == 3))) + return; + + rw = tsk->arg & (1 << 30); /* write: 0, read: 1 */ + bytes = (tsk->arg & 0xFFFF) << SECTOR_SHIFT; + + mtk_btag_task_timetag(buf, 256, stage, tsk_max, task_name, tsk->t, + bytes); + + pr_debug("[BLOCK_TAG] cmdq: tsk[%d],%d,%s,len=%d%s\n", + tsk->task_id, stage+1, (rw)?"r":"w", bytes, buf); +} + +void mt_biolog_cmdq_check(void) +{ + struct mt_bio_context *ctx; + __u64 end_time, period_time; + unsigned long flags; + + ctx = mt_bio_curr_ctx(); + if (!ctx) + return; + + spin_lock_irqsave(&ctx->lock, flags); + end_time = sched_clock(); + period_time = end_time - ctx->period_start_t; + + if (period_time >= MT_BIO_TRACE_LATENCY) { + ctx->period_end_t = end_time; + ctx->workload.period = period_time; + mt_bio_context_eval(ctx); + mt_bio_print_trace(ctx); + ctx->period_start_t = end_time; + ctx->period_end_t = 0; + ctx->wl.period_busy = 0; + ctx->wl.period_left_window_end_t = 0; + ctx->wl.period_right_window_end_t = 0; + ctx->wl.period_right_window_start_t = 0; + memset(&ctx->throughput, 0, sizeof(struct mtk_btag_throughput)); + memset(&ctx->workload, 0, sizeof(struct mtk_btag_workload)); + } + spin_unlock_irqrestore(&ctx->lock, flags); +} + +/* Command Queue Hook: stage1: queue task */ +void mt_biolog_cmdq_queue_task(unsigned int task_id, struct mmc_request *req) +{ + struct mt_bio_context *ctx; + struct mt_bio_context_task *tsk; + int i; + + if (!req) + return; + if (!req->sbc) + return; + + tsk = mt_bio_curr_task(task_id, &ctx); + if (!tsk) + return; + + if (ctx->state == CMDQ_CTX_NOT_DMA) + ctx->state = CMDQ_CTX_QUEUE; + + tsk->arg = req->sbc->arg; + tsk->t[tsk_req_start] = sched_clock(); + + ctx->q_depth++; + mtk_btag_mictx_update_ctx(ctx->q_depth); + + for (i = tsk_dma_start; i < tsk_max; i++) + tsk->t[i] = 0; + + if (!ctx->period_start_t) + ctx->period_start_t = tsk->t[tsk_req_start]; + + mt_pr_cmdq_tsk(tsk, tsk_req_start); +} + +static void mt_bio_ctx_count_usage(struct mt_bio_context *ctx, + __u64 start, __u64 end) +{ + if (start <= ctx->period_start_t) { + /* + * if 'start' is located in previous period, + * reset right window and period_busy, + * and finally only left window will be existed. + */ + ctx->wl.period_left_window_end_t = end; + ctx->wl.period_right_window_start_t = + ctx->wl.period_right_window_end_t = + ctx->wl.period_busy = 0; + } else { + /* if left window is existed */ + if (ctx->wl.period_left_window_end_t) { + if (start < ctx->wl.period_left_window_end_t) { + /* + * if 'start' is located inside left window, + * reset right window and period_busy, + * and finally only left window will be existed. + */ + ctx->wl.period_left_window_end_t = end; + ctx->wl.period_right_window_start_t = + ctx->wl.period_right_window_end_t = + ctx->wl.period_busy = 0; + } else + goto new_window; + } else + goto new_window; + } + + goto out; + +new_window: + + if (ctx->wl.period_right_window_start_t) { + if (start > ctx->wl.period_right_window_end_t) { + ctx->wl.period_busy += + (ctx->wl.period_right_window_end_t - + ctx->wl.period_right_window_start_t); + ctx->wl.period_right_window_start_t = start; + } + ctx->wl.period_right_window_end_t = end; + } else { + ctx->wl.period_right_window_start_t = start; + ctx->wl.period_right_window_end_t = end; + } + +out: + return; +} + +static uint64_t mt_bio_get_period_busy(struct mt_bio_context *ctx) +{ + uint64_t busy; + + busy = ctx->wl.period_busy; + + if (ctx->wl.period_left_window_end_t) { + busy += + (ctx->wl.period_left_window_end_t - + ctx->period_start_t); + } + + if (ctx->wl.period_right_window_start_t) { + busy += + (ctx->wl.period_right_window_end_t - + ctx->wl.period_right_window_start_t); + } + + return busy; +} + +/* Command Queue Hook: stage2: dma start */ +void mt_biolog_cmdq_dma_start(unsigned int task_id) +{ + struct mt_bio_context_task *tsk; + struct mt_bio_context *ctx; + + tsk = mt_bio_curr_task(task_id, &ctx); + if (!tsk) + return; + tsk->t[tsk_dma_start] = sched_clock(); + + /* count first queue task time in workload usage, + * if it was not overlapped with DMA + */ + if (ctx->state == CMDQ_CTX_QUEUE) + mt_bio_ctx_count_usage(ctx, tsk->t[tsk_req_start], + tsk->t[tsk_dma_start]); + ctx->state = CMDQ_CTX_IN_DMA; + mt_pr_cmdq_tsk(tsk, tsk_dma_start); +} + +/* Command Queue Hook: stage3: dma end */ +void mt_biolog_cmdq_dma_end(unsigned int task_id) +{ + struct mt_bio_context_task *tsk; + struct mt_bio_context *ctx; + + tsk = mt_bio_curr_task(task_id, &ctx); + if (!tsk) + return; + tsk->t[tsk_dma_end] = sched_clock(); + ctx->state = CMDQ_CTX_NOT_DMA; + mt_pr_cmdq_tsk(tsk, tsk_dma_end); +} + +/* Command Queue Hook: stage4: isdone start */ +void mt_biolog_cmdq_isdone_start(unsigned int task_id, struct mmc_request *req) +{ + struct mt_bio_context_task *tsk; + + tsk = mt_bio_curr_task(task_id, NULL); + if (!tsk) + return; + tsk->t[tsk_isdone_start] = sched_clock(); + mt_pr_cmdq_tsk(tsk, tsk_isdone_start); +} + +/* Command Queue Hook: stage5: isdone end */ +void mt_biolog_cmdq_isdone_end(unsigned int task_id) +{ + int write, i; + __u32 bytes; + __u64 end_time, busy_time; + struct mt_bio_context *ctx; + struct mt_bio_context_task *tsk; + struct mtk_btag_throughput_rw *tp; + + tsk = mt_bio_curr_task(task_id, &ctx); + if (!tsk) + return; + + /* return if there's no on-going request */ + for (i = 0; i < tsk_isdone_end; i++) + if (!tsk->t[i]) + return; + + tsk->t[tsk_isdone_end] = end_time = sched_clock(); + + ctx->q_depth--; + mtk_btag_mictx_update_ctx(ctx->q_depth); + + /* throughput usage := duration of handling this request */ + + /* tsk->arg & (1 << 30): 0 means write */ + write = tsk->arg & (1 << 30); + write = (write) ? 0 : 1; + + bytes = tsk->arg & 0xFFFF; + bytes = bytes << SECTOR_SHIFT; + busy_time = end_time - tsk->t[tsk_req_start]; + tp = (write) ? &ctx->throughput.w : &ctx->throughput.r; + tp->usage += busy_time; + tp->size += bytes; + + mtk_btag_mictx_eval_tp(write, busy_time, bytes); + + /* workload statistics */ + ctx->workload.count++; + + /* count DMA time in workload usage */ + + /* count isdone time in workload usage, + * if it was not overlapped with DMA + */ + if (ctx->state == CMDQ_CTX_NOT_DMA) + mt_bio_ctx_count_usage(ctx, tsk->t[tsk_dma_start], end_time); + else + mt_bio_ctx_count_usage(ctx, tsk->t[tsk_dma_start], + tsk->t[tsk_dma_end]); + + mt_pr_cmdq_tsk(tsk, tsk_isdone_end); + + mt_bio_init_task(tsk); +} + +void mt_biolog_cqhci_check(void) +{ + mt_biolog_cmdq_check(); +} + +void mt_biolog_cqhci_queue_task(unsigned int task_id, struct mmc_request *req) +{ + struct mt_bio_context *ctx; + struct mt_bio_context_task *tsk; + unsigned long flags; + + if (!req) + return; + + tsk = mt_bio_curr_task_by_ctx_id(task_id, + &ctx, CTX_MMCCMDQD0); + if (!tsk) + return; + + spin_lock_irqsave(&ctx->lock, flags); + +#ifdef CONFIG_MTK_EMMC_HW_CQ + /* convert cqhci to legacy sbc arg */ + tsk->arg = (!!(req->cmdq_req->cmdq_req_flags & DIR)) << 30 | + (req->cmdq_req->data.blocks & 0xFFFF); +#else + if (req->sbc) + tsk->arg = req->sbc->arg; +#endif + + tsk->t[tsk_req_start] = sched_clock(); + + ctx->q_depth++; + mtk_btag_mictx_update_ctx(ctx->q_depth); + + if (!ctx->period_start_t) + ctx->period_start_t = tsk->t[tsk_req_start]; + + spin_unlock_irqrestore(&ctx->lock, flags); + + mt_pr_cmdq_tsk(tsk, tsk_req_start); +} + +void mt_biolog_cqhci_complete(unsigned int task_id) +{ + int write; + __u32 bytes; + __u64 end_time, busy_time; + struct mt_bio_context *ctx; + struct mt_bio_context_task *tsk; + struct mtk_btag_throughput_rw *tp; + unsigned long flags; + + tsk = mt_bio_curr_task_by_ctx_id(task_id, + &ctx, CTX_MMCCMDQD0); + if (!tsk) + return; + + spin_lock_irqsave(&ctx->lock, flags); + + tsk->t[tsk_isdone_end] = end_time = sched_clock(); + + ctx->q_depth--; + mtk_btag_mictx_update_ctx(ctx->q_depth); + + /* throughput usage := duration of handling this request */ + + /* tsk->arg & (1 << 30): 0 means write */ + write = tsk->arg & (1 << 30); + write = (write) ? 0 : 1; + + bytes = tsk->arg & 0xFFFF; + bytes = bytes << SECTOR_SHIFT; + busy_time = end_time - tsk->t[tsk_req_start]; + + tp = (write) ? &ctx->throughput.w : &ctx->throughput.r; + tp->usage += busy_time; + tp->size += bytes; + + mtk_btag_mictx_eval_tp(write, busy_time, bytes); + + /* workload statistics */ + ctx->workload.count++; + + /* count doorbell to complete time in workload usage */ + mt_bio_ctx_count_usage(ctx, tsk->t[tsk_req_start], end_time); + + mt_pr_cmdq_tsk(tsk, tsk_isdone_end); + + mt_bio_init_task(tsk); + spin_unlock_irqrestore(&ctx->lock, flags); +} + +/* MMC Queue Hook: check function at mmc_blk_issue_rw_rq() */ +void mt_biolog_mmcqd_req_check(void) +{ + struct mt_bio_context *ctx; + __u64 end_time, period_time; + + ctx = mt_bio_curr_ctx(); + if (!ctx) + return; + + /* skip mmcqd0, if command queue is applied */ + if ((ctx->id == CTX_MMCQD0) && (ctx->state == MMCQD_CMDQ_MODE_EN)) + return; + + end_time = sched_clock(); + period_time = end_time - ctx->period_start_t; + + if (period_time >= MT_BIO_TRACE_LATENCY) { + ctx->period_end_t = end_time; + ctx->workload.period = period_time; + mt_bio_context_eval(ctx); + mt_bio_print_trace(ctx); + ctx->period_start_t = end_time; + ctx->period_end_t = 0; + ctx->wl.period_busy = 0; + ctx->wl.period_left_window_end_t = 0; + ctx->wl.period_right_window_end_t = 0; + ctx->wl.period_right_window_start_t = 0; + memset(&ctx->throughput, 0, sizeof(struct mtk_btag_throughput)); + memset(&ctx->workload, 0, sizeof(struct mtk_btag_workload)); + } +} + +/* MMC Queue Hook: request start function at mmc_start_req() */ +void mt_biolog_mmcqd_req_start(struct mmc_host *host) +{ + struct mt_bio_context *ctx; + struct mt_bio_context_task *tsk; + + tsk = mt_bio_curr_task(0, &ctx); + if (!tsk) + return; + tsk->t[tsk_req_start] = sched_clock(); + +#ifdef CONFIG_MTK_EMMC_CQ_SUPPORT + if ((ctx->id == CTX_MMCQD0) && + (ctx->state == MMCQD_NORMAL) && + host->card->ext_csd.cmdq_en) + ctx->state = MMCQD_CMDQ_MODE_EN; + + /* + * CMDQ mode, embedded eMMC mictx will be + * updated in mt_biolog_cmdq_*, so bypass it here. + */ +#else + /* Legacy mode. Update mictx for embedded eMMC only */ + if (ctx->qid == BTAG_STORAGE_EMBEDDED) + mtk_btag_mictx_update_ctx(1); +#endif +} + +/* MMC Queue Hook: request end function at mmc_start_req() */ +void mt_biolog_mmcqd_req_end(struct mmc_data *data) +{ + int rw; + __u32 size; + struct mt_bio_context *ctx; + struct mt_bio_context_task *tsk; + struct mtk_btag_throughput_rw *tp; + __u64 end_time, busy_time; + + end_time = sched_clock(); + + if (!data) + return; + + if (data->flags == MMC_DATA_WRITE) + rw = 0; + else if (data->flags == MMC_DATA_READ) + rw = 1; + else + return; + + tsk = mt_bio_curr_task(0, &ctx); + if (!tsk) + return; + + /* return if there's no on-going request */ + if (!tsk->t[tsk_req_start]) + return; + + size = (data->blocks * data->blksz); + busy_time = end_time - tsk->t[tsk_req_start]; + + /* workload statistics */ + ctx->workload.count++; + + /* count request handling time in workload usage */ + mt_bio_ctx_count_usage(ctx, tsk->t[tsk_req_start], end_time); + + /* throughput statistics */ + /* write: 0, read: 1 */ + tp = (rw) ? &ctx->throughput.r : &ctx->throughput.w; + tp->usage += busy_time; + tp->size += size; + + /* update mictx for embedded eMMC only */ + if (ctx->qid == BTAG_STORAGE_EMBEDDED) { + mtk_btag_mictx_eval_tp(!rw, busy_time, size); + mtk_btag_mictx_update_ctx(0); + } + + /* re-init task to indicate no on-going request */ + mt_bio_init_task(tsk); +} + +/* + * snprintf may return a value of size or "more" to indicate + * that the output was truncated, thus be careful of "more" + * case. + */ +#define SPREAD_PRINTF(buff, size, evt, fmt, args...) \ + do { \ + if (buff && size && *(size)) { \ + unsigned long var = snprintf(*(buff), *(size),\ + fmt, ##args); \ + if (var > 0) { \ + if (var > *(size)) \ + var = *(size); \ + *(size) -= var; \ + *(buff) += var; \ + } \ + } \ + if (evt) \ + seq_printf(evt, fmt, ##args); \ + if (!buff && !evt) { \ + pr_info(fmt, ##args); \ + } \ + } while (0) + +static size_t mt_bio_seq_debug_show_info(char **buff, unsigned long *size, + struct seq_file *seq) +{ + int i; + struct mt_bio_context *ctx = BTAG_CTX(mtk_btag_mmc); + + if (!ctx) + return 0; + + for (i = 0; i < MMC_BIOLOG_CONTEXTS; i++) { + if (ctx[i].pid == 0) + continue; + SPREAD_PRINTF(buff, size, seq, + "ctx[%d]=ctx_map[%d]=%s,pid:%4d,q:%d\n", + i, + ctx[i].id, + ctx[i].comm, + ctx[i].pid, + ctx[i].qid); + } + + return 0; +} + +int mt_mmc_biolog_init(void) +{ + struct mtk_blocktag *btag; + + btag = mtk_btag_alloc("mmc", + MMC_BIOLOG_RINGBUF_MAX, + sizeof(struct mt_bio_context), + MMC_BIOLOG_CONTEXTS, + mt_bio_seq_debug_show_info); + + if (btag) + mtk_btag_mmc = btag; + + return 0; +} +EXPORT_SYMBOL_GPL(mt_mmc_biolog_init); + +int mt_mmc_biolog_exit(void) +{ + mtk_btag_free(mtk_btag_mmc); + return 0; +} + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Mediatek MMC Block IO Log"); + diff --git a/drivers/mmc/core/mtk_mmc_block.h b/drivers/mmc/core/mtk_mmc_block.h new file mode 100644 index 000000000000..303ccf02069c --- /dev/null +++ b/drivers/mmc/core/mtk_mmc_block.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 MediaTek Inc. + */ + +#ifndef _MT_MMC_BLOCK_H +#define _MT_MMC_BLOCK_H + +#include +#include +#include +#include +#include "../../misc/mediatek/include/mt-plat/mtk_blocktag.h" + +#if defined(CONFIG_MMC_BLOCK_IO_LOG) + +int mt_mmc_biolog_init(void); +int mt_mmc_biolog_exit(void); + +void mt_bio_queue_alloc(struct task_struct *thread, struct request_queue *q); +void mt_bio_queue_free(struct task_struct *thread); + +void mt_biolog_mmcqd_req_check(void); +void mt_biolog_mmcqd_req_start(struct mmc_host *host); +void mt_biolog_mmcqd_req_end(struct mmc_data *data); + +void mt_biolog_cmdq_check(void); +void mt_biolog_cmdq_queue_task(unsigned int task_id, struct mmc_request *req); +void mt_biolog_cmdq_dma_start(unsigned int task_id); +void mt_biolog_cmdq_dma_end(unsigned int task_id); +void mt_biolog_cmdq_isdone_start(unsigned int task_id, struct mmc_request *req); +void mt_biolog_cmdq_isdone_end(unsigned int task_id); + +void mt_biolog_cqhci_check(void); +void mt_biolog_cqhci_queue_task(unsigned int task_id, struct mmc_request *req); +void mt_biolog_cqhci_complete(unsigned int task_id); + +#define MMC_BIOLOG_RINGBUF_MAX 120 +#define MMC_BIOLOG_CONTEXTS 10 /* number of request queues */ +#define MMC_BIOLOG_CONTEXT_TASKS 32 /* number concurrent tasks in cmdq */ + +enum { + tsk_req_start = 0, + tsk_dma_start, + tsk_dma_end, + tsk_isdone_start, + tsk_isdone_end, + tsk_max +}; + +struct mt_bio_context_task { + int task_id; + u32 arg; + uint64_t t[tsk_max]; +}; + +struct mt_bio_context_wl { + uint64_t period_busy; + uint64_t period_left_window_end_t; + uint64_t period_right_window_start_t; + uint64_t period_right_window_end_t; +}; + +/* Context of Request Queue */ +struct mt_bio_context { + int id; + int state; + pid_t pid; + struct request_queue *q; + char comm[TASK_COMM_LEN]; + u16 qid; + u16 q_depth; + spinlock_t lock; + uint64_t period_start_t; + uint64_t period_end_t; + struct mt_bio_context_wl wl; + struct mt_bio_context_task task[MMC_BIOLOG_CONTEXT_TASKS]; + struct mtk_btag_workload workload; + struct mtk_btag_throughput throughput; + struct mtk_btag_pidlogger pidlog; +}; + +#else + +#define mt_mmc_biolog_init(...) +#define mt_mmc_biolog_exit(...) + +#define mt_bio_queue_alloc(...) +#define mt_bio_queue_free(...) + +#define mt_biolog_mmcqd_req_check(...) +#define mt_biolog_mmcqd_req_start(...) +#define mt_biolog_mmcqd_req_end(...) + +#define mt_biolog_cmdq_check(...) +#define mt_biolog_cmdq_queue_task(...) +#define mt_biolog_cmdq_dma_start(...) +#define mt_biolog_cmdq_dma_end(...) +#define mt_biolog_cmdq_isdone_start(...) +#define mt_biolog_cmdq_isdone_end(...) + +#define mt_biolog_cqhci_check(...) +#define mt_biolog_cqhci_queue_task(...) +#define mt_biolog_cqhci_complete(...) + +#endif + +#endif diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index 8b1f3c8d5cf4..6d787ef44393 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -24,6 +24,7 @@ #include "card.h" #include "host.h" #include "mmc-crypto.h" +#include "mtk_mmc_block.h" static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq) { @@ -288,6 +289,7 @@ static blk_status_t mmc_mq_queue_rq(struct blk_mq_hw_ctx *hctx, break; } + mt_biolog_mmcqd_req_check(); /* Parallel dispatch of requests is not supported at the moment */ mq->busy = true; diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 078f1461e074..469cfef3cf1d 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "internal.h" @@ -2427,6 +2428,13 @@ void account_page_dirtied(struct page *page, struct address_space *mapping) task_io_account_write(PAGE_SIZE); current->nr_dirtied++; this_cpu_inc(bdp_ratelimits); + + /* + * Dirty pages may be written by writeback thread later. + * To get real i/o owner of this page, we shall keep it + * before writeback takes over. + */ + mtk_btag_pidlog_set_pid(page); } } EXPORT_SYMBOL(account_page_dirtied);