blob: 2139655e6d4a9708970ad43b081ad85ba169784c [file] [log] [blame]
/*
* Copyright (C) 2007 Google, Inc.
* Copyright (c) 2012-2017, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*/
#include "msm_qpic_nand.h"
#define QPIC_BAM_DEFAULT_IPC_LOGLVL 2
/* The driver supports devices upto 4K page */
#define MAX_CW_PER_PAGE 8
/*
* Max descriptors needed for erase, read, write operations.
* Usually, this is (2 * MAX_CW_PER_PAGE).
*/
#define MAX_DESC 16
static bool enable_euclean;
static bool enable_perfstats;
static ssize_t msm_nand_attr_perf_stats_show(struct device *dev,
struct device_attribute *attr,
char *buf);
static ssize_t msm_nand_attr_perf_stats_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count);
static struct device_attribute dev_msm_nand_perf_stats =
__ATTR(perf_stats, 0644,
msm_nand_attr_perf_stats_show, msm_nand_attr_perf_stats_store);
#define print_sysfs(fmt, ...) \
{ \
count += scnprintf(buf + count, PAGE_SIZE - count, \
fmt, ##__VA_ARGS__); \
}
static ssize_t msm_nand_attr_perf_stats_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
ssize_t count = 0;
struct msm_nand_info *info = dev_get_drvdata(dev);
if (!enable_perfstats) {
print_sysfs("Performance stats is disabled\n");
return count;
}
spin_lock(&info->perf.lock);
print_sysfs("total_read_size = %llu\n", info->perf.total_read_size);
print_sysfs("total_write_size = %llu\n", info->perf.total_write_size);
print_sysfs("total_erase_blks = %llu\n\n", info->perf.total_erase_blks);
print_sysfs("total_read_time_us = %lld\n",
ktime_to_us(info->perf.total_read_time));
print_sysfs("total_write_time_us = %lld\n",
ktime_to_us(info->perf.total_write_time));
print_sysfs("total_erase_time_us = %lld\n\n",
ktime_to_us(info->perf.total_erase_time));
print_sysfs("min_read_time_us = %lld\n",
ktime_to_us(info->perf.min_read_time));
print_sysfs("min_write_time_us = %lld\n",
ktime_to_us(info->perf.min_write_time));
print_sysfs("min_erase_time_us = %lld\n\n",
ktime_to_us(info->perf.min_erase_time));
print_sysfs("max_read_time_us = %lld\n",
ktime_to_us(info->perf.max_read_time));
print_sysfs("max_write_time_us = %lld\n",
ktime_to_us(info->perf.max_write_time));
print_sysfs("max_erase_time_us = %lld\n\n",
ktime_to_us(info->perf.max_erase_time));
spin_unlock(&info->perf.lock);
return count;
}
static ssize_t msm_nand_attr_perf_stats_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct msm_nand_info *info = dev_get_drvdata(dev);
if (!enable_perfstats) {
pr_err("couldn't write as perf stats is disabled\n");
return -EPERM;
}
if (count > 1 || (count == 1 && *buf != '\n')) {
pr_err("write not permitted\n");
return -EPERM;
}
spin_lock(&info->perf.lock);
info->perf.min_read_time = ktime_set(KTIME_MAX, 0);
info->perf.min_write_time = ktime_set(KTIME_MAX, 0);
info->perf.min_erase_time = ktime_set(KTIME_MAX, 0);
info->perf.max_read_time = ktime_set(0, 0);
info->perf.max_write_time = ktime_set(0, 0);
info->perf.max_erase_time = ktime_set(0, 0);
info->perf.total_read_time = ktime_set(0, 0);
info->perf.total_write_time = ktime_set(0, 0);
info->perf.total_erase_time = ktime_set(0, 0);
info->perf.total_read_size = 0;
info->perf.total_write_size = 0;
info->perf.total_erase_blks = 0;
spin_unlock(&info->perf.lock);
return count;
}
static void msm_nand_init_perf_stats(struct msm_nand_info *info)
{
spin_lock_init(&info->perf.lock);
info->perf.min_read_time = ktime_set(KTIME_MAX, 0);
info->perf.min_write_time = ktime_set(KTIME_MAX, 0);
info->perf.min_erase_time = ktime_set(KTIME_MAX, 0);
}
static void msm_nand_init_sysfs(struct device *dev)
{
sysfs_attr_init(&dev_msm_nand_perf_stats);
if (device_create_file(dev, &dev_msm_nand_perf_stats))
pr_err("Sysfs entry create failed");
}
static void msm_nand_cleanup_sysfs(struct device *dev)
{
device_remove_file(dev, &dev_msm_nand_perf_stats);
}
static void msm_nand_update_read_perf_stats(struct msm_nand_info *info,
ktime_t start, u32 size)
{
ktime_t time_delta;
time_delta = ktime_sub(ktime_get(), start);
spin_lock(&info->perf.lock);
info->perf.total_read_size += size;
info->perf.total_read_time = ktime_add(info->perf.total_read_time,
time_delta);
if (ktime_after(time_delta, info->perf.max_read_time))
info->perf.max_read_time = time_delta;
if (ktime_before(time_delta, info->perf.min_read_time))
info->perf.min_read_time = time_delta;
spin_unlock(&info->perf.lock);
}
static void msm_nand_update_write_perf_stats(struct msm_nand_info *info,
ktime_t start, u32 size)
{
ktime_t time_delta;
time_delta = ktime_sub(ktime_get(), start);
spin_lock(&info->perf.lock);
info->perf.total_write_size += size;
info->perf.total_write_time = ktime_add(info->perf.total_write_time,
time_delta);
if (ktime_after(time_delta, info->perf.max_write_time))
info->perf.max_write_time = time_delta;
if (ktime_before(time_delta, info->perf.min_write_time))
info->perf.min_write_time = time_delta;
spin_unlock(&info->perf.lock);
}
static void msm_nand_update_erase_perf_stats(struct msm_nand_info *info,
ktime_t start, u32 count)
{
ktime_t time_delta;
time_delta = ktime_sub(ktime_get(), start);
spin_lock(&info->perf.lock);
info->perf.total_erase_blks += count;
info->perf.total_erase_time = ktime_add(info->perf.total_erase_time,
time_delta);
if (ktime_after(time_delta, info->perf.max_erase_time))
info->perf.max_erase_time = time_delta;
if (ktime_before(time_delta, info->perf.min_erase_time))
info->perf.min_erase_time = time_delta;
spin_unlock(&info->perf.lock);
}
/*
* Get the DMA memory for requested amount of size. It returns the pointer
* to free memory available from the allocated pool. Returns NULL if there
* is no free memory.
*/
static void *msm_nand_get_dma_buffer(struct msm_nand_chip *chip, size_t size)
{
uint32_t bitmask, free_bitmask, old_bitmask;
uint32_t need_mask, current_need_mask;
int free_index;
need_mask = (1UL << DIV_ROUND_UP(size, MSM_NAND_DMA_BUFFER_SLOT_SZ))
- 1;
bitmask = atomic_read(&chip->dma_buffer_busy);
free_bitmask = ~bitmask;
if (free_bitmask == 0)
return NULL;
do {
free_index = __ffs(free_bitmask);
current_need_mask = need_mask << free_index;
if (size + free_index * MSM_NAND_DMA_BUFFER_SLOT_SZ >=
MSM_NAND_DMA_BUFFER_SIZE)
return NULL;
if ((bitmask & current_need_mask) == 0) {
old_bitmask =
atomic_cmpxchg(&chip->dma_buffer_busy,
bitmask,
bitmask | current_need_mask);
if (old_bitmask == bitmask)
return chip->dma_virt_addr +
free_index * MSM_NAND_DMA_BUFFER_SLOT_SZ;
free_bitmask = 0;/* force return */
}
/* current free range was too small, clear all free bits */
/* below the top busy bit within current_need_mask */
free_bitmask &=
~(~0U >> (32 - fls(bitmask & current_need_mask)));
} while (free_bitmask);
return NULL;
}
/*
* Releases the DMA memory used to the free pool and also wakes up any user
* thread waiting on wait queue for free memory to be available.
*/
static void msm_nand_release_dma_buffer(struct msm_nand_chip *chip,
void *buffer, size_t size)
{
int index;
uint32_t used_mask;
used_mask = (1UL << DIV_ROUND_UP(size, MSM_NAND_DMA_BUFFER_SLOT_SZ))
- 1;
index = ((uint8_t *)buffer - chip->dma_virt_addr) /
MSM_NAND_DMA_BUFFER_SLOT_SZ;
atomic_sub(used_mask << index, &chip->dma_buffer_busy);
wake_up(&chip->dma_wait_queue);
}
/*
* Calculates page address of the buffer passed, offset of buffer within
* that page and then maps it for DMA by calling dma_map_page().
*/
static dma_addr_t msm_nand_dma_map(struct device *dev, void *addr, size_t size,
enum dma_data_direction dir)
{
struct page *page;
unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
if (virt_addr_valid(addr))
page = virt_to_page(addr);
else {
if (WARN_ON(size + offset > PAGE_SIZE))
return ~0;
page = vmalloc_to_page(addr);
}
return dma_map_page(dev, page, offset, size, dir);
}
#ifdef CONFIG_QCOM_BUS_SCALING
static int msm_nand_bus_set_vote(struct msm_nand_info *info,
unsigned int vote)
{
int ret = 0;
ret = msm_bus_scale_client_update_request(info->clk_data.client_handle,
vote);
if (ret)
pr_err("msm_bus_scale_client_update_request() failed, bus_client_handle=0x%x, vote=%d, err=%d\n",
info->clk_data.client_handle, vote, ret);
return ret;
}
static int msm_nand_setup_clocks_and_bus_bw(struct msm_nand_info *info,
bool vote)
{
int ret = 0;
if (!info->clk_data.rpmh_clk) {
if (IS_ERR_OR_NULL(info->clk_data.qpic_clk)) {
ret = -EINVAL;
goto out;
}
}
if (atomic_read(&info->clk_data.clk_enabled) == vote)
goto out;
if (!atomic_read(&info->clk_data.clk_enabled) && vote) {
ret = msm_nand_bus_set_vote(info, 1);
if (ret) {
pr_err("Failed to vote for bus with %d\n", ret);
goto out;
}
if (!info->clk_data.rpmh_clk) {
ret = clk_prepare_enable(info->clk_data.qpic_clk);
if (ret) {
pr_err("Failed to enable the bus-clock with error %d\n",
ret);
msm_nand_bus_set_vote(info, 0);
goto out;
}
}
} else if (atomic_read(&info->clk_data.clk_enabled) && !vote) {
if (!info->clk_data.rpmh_clk)
clk_disable_unprepare(info->clk_data.qpic_clk);
msm_nand_bus_set_vote(info, 0);
}
atomic_set(&info->clk_data.clk_enabled, vote);
out:
return ret;
}
#else
static int msm_nand_setup_clocks_and_bus_bw(struct msm_nand_info *info,
bool vote)
{
return 0;
}
#endif
#ifdef CONFIG_PM
static int msm_nand_runtime_suspend(struct device *dev)
{
int ret = 0;
struct msm_nand_info *info = dev_get_drvdata(dev);
ret = msm_nand_setup_clocks_and_bus_bw(info, false);
return ret;
}
static int msm_nand_runtime_resume(struct device *dev)
{
int ret = 0;
struct msm_nand_info *info = dev_get_drvdata(dev);
ret = msm_nand_setup_clocks_and_bus_bw(info, true);
return ret;
}
static void msm_nand_print_rpm_info(struct device *dev)
{
pr_err("RPM: runtime_status=%d, usage_count=%d, is_suspended=%d, disable_depth=%d, runtime_error=%d, request_pending=%d, request=%d\n",
dev->power.runtime_status, atomic_read(&dev->power.usage_count),
dev->power.is_suspended, dev->power.disable_depth,
dev->power.runtime_error, dev->power.request_pending,
dev->power.request);
}
#else
static int msm_nand_runtime_suspend(struct device *dev)
{
return 0;
}
static int msm_nand_runtime_resume(struct device *dev)
{
return 0;
}
static void msm_nand_print_rpm_info(struct device *dev)
{
}
#endif
#ifdef CONFIG_PM
static int msm_nand_suspend(struct device *dev)
{
int ret = 0;
if (!pm_runtime_suspended(dev))
ret = msm_nand_runtime_suspend(dev);
return ret;
}
static int msm_nand_resume(struct device *dev)
{
int ret = 0;
if (!pm_runtime_suspended(dev))
ret = msm_nand_runtime_resume(dev);
return ret;
}
#else
static int msm_nand_suspend(struct device *dev)
{
return 0;
}
static int msm_nand_resume(struct device *dev)
{
return 0;
}
#endif
#ifdef CONFIG_PM
static int msm_nand_get_device(struct device *dev)
{
int ret = 0;
ret = pm_runtime_get_sync(dev);
if (ret < 0) {
pr_err("Failed to resume with %d\n", ret);
msm_nand_print_rpm_info(dev);
} else { /* Reset to success */
ret = 0;
}
return ret;
}
static int msm_nand_put_device(struct device *dev)
{
int ret = 0;
pm_runtime_mark_last_busy(dev);
ret = pm_runtime_put_autosuspend(dev);
if (ret < 0) {
pr_err("Failed to suspend with %d\n", ret);
msm_nand_print_rpm_info(dev);
} else { /* Reset to success */
ret = 0;
}
return ret;
}
#else
static int msm_nand_get_device(struct device *dev)
{
return 0;
}
static int msm_nand_put_device(struct device *dev)
{
return 0;
}
#endif
#ifdef CONFIG_QCOM_BUS_SCALING
static int msm_nand_bus_register(struct platform_device *pdev,
struct msm_nand_info *info)
{
int ret = 0;
info->clk_data.use_cases = msm_bus_cl_get_pdata(pdev);
if (!info->clk_data.use_cases) {
ret = -EINVAL;
pr_err("msm_bus_cl_get_pdata failed\n");
goto out;
}
info->clk_data.client_handle =
msm_bus_scale_register_client(info->clk_data.use_cases);
if (!info->clk_data.client_handle) {
ret = -EINVAL;
pr_err("msm_bus_scale_register_client failed\n");
}
out:
return ret;
}
static void msm_nand_bus_unregister(struct msm_nand_info *info)
{
if (info->clk_data.client_handle)
msm_bus_scale_unregister_client(info->clk_data.client_handle);
}
#else
static int msm_nand_bus_register(struct platform_device *pdev,
struct msm_nand_info *info)
{
pr_info("couldn't register due to missing config option\n");
return 0;
}
static void msm_nand_bus_unregister(struct msm_nand_info *info)
{
}
#endif
/*
* Wrapper function to prepare a single SPS command element with the data
* that is passed to this function.
*/
static inline void msm_nand_prep_ce(struct sps_command_element *ce,
uint32_t addr, uint32_t command, uint32_t data)
{
ce->addr = addr;
ce->command = (command & WRITE) ? (uint32_t) SPS_WRITE_COMMAND :
(uint32_t) SPS_READ_COMMAND;
ce->data = data;
ce->mask = 0xFFFFFFFF;
}
static int msm_nand_sps_get_iovec(struct sps_pipe *pipe, uint32_t indx,
unsigned int cnt, struct sps_iovec *iovec)
{
int ret = 0;
do {
do {
ret = sps_get_iovec((pipe), (iovec));
} while (((iovec)->addr == 0x0) && ((iovec)->size == 0x0));
if (ret)
return ret;
} while (--(cnt));
return ret;
}
/*
* Wrapper function to prepare a single command descriptor with a single
* SPS command element with the data that is passed to this function.
*
* Since for any command element it is a must to have this flag
* SPS_IOVEC_FLAG_CMD, this function by default updates this flag for a
* command element that is passed and thus, the caller need not explicilty
* pass this flag. The other flags must be passed based on the need. If a
* command element doesn't have any other flag, then 0 can be passed to flags.
*/
static inline void msm_nand_prep_single_desc(struct msm_nand_sps_cmd *sps_cmd,
uint32_t addr, uint32_t command,
uint32_t data, uint32_t flags)
{
msm_nand_prep_ce(&sps_cmd->ce, addr, command, data);
sps_cmd->flags = SPS_IOVEC_FLAG_CMD | flags;
}
/*
* Read a single NANDc register as mentioned by its parameter addr. The return
* value indicates whether read is successful or not. The register value read
* is stored in val.
*/
static int msm_nand_flash_rd_reg(struct msm_nand_info *info, uint32_t addr,
uint32_t *val)
{
int ret = 0, submitted_num_desc = 1;
struct msm_nand_sps_cmd *cmd;
struct msm_nand_chip *chip = &info->nand_chip;
struct {
struct msm_nand_sps_cmd cmd;
uint32_t data;
} *dma_buffer;
struct sps_iovec iovec_temp;
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer(
chip, sizeof(*dma_buffer))));
cmd = &dma_buffer->cmd;
msm_nand_prep_single_desc(cmd, addr, READ, msm_virt_to_dma(chip,
&dma_buffer->data), SPS_IOVEC_FLAG_INT);
mutex_lock(&info->lock);
ret = msm_nand_get_device(chip->dev);
if (ret)
goto out;
ret = sps_transfer_one(info->sps.cmd_pipe.handle,
msm_virt_to_dma(chip, &cmd->ce),
sizeof(struct sps_command_element), NULL, cmd->flags);
if (ret) {
pr_err("failed to submit command %x ret %d\n", addr, ret);
msm_nand_put_device(chip->dev);
goto out;
}
ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index, submitted_num_desc,
&iovec_temp);
if (ret) {
pr_err("Failed to get iovec for pipe %d: (ret%d)\n",
(info->sps.cmd_pipe.index), ret);
goto out;
}
ret = msm_nand_put_device(chip->dev);
if (ret)
goto out;
*val = dma_buffer->data;
out:
mutex_unlock(&info->lock);
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
return ret;
}
/*
* Read the Flash ID from the Nand Flash Device. The return value < 0
* indicates failure. When successful, the Flash ID is stored in parameter
* read_id.
*/
#define READID_CMDS 5
static int msm_nand_flash_read_id(struct msm_nand_info *info,
bool read_onfi_signature, uint32_t *read_id,
uint32_t *read_id2)
{
int err = 0, i = 0;
struct msm_nand_sps_cmd *cmd;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
struct msm_nand_chip *chip = &info->nand_chip;
/*
* The following 5 commands are required to read id -
* write commands - addr0, flash, exec
* read_commands - read_id, read_id2
*/
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[READID_CMDS];
struct msm_nand_sps_cmd cmd[READID_CMDS];
uint32_t data[READID_CMDS];
} *dma_buffer;
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer
(chip, sizeof(*dma_buffer))));
if (read_onfi_signature)
dma_buffer->data[0] = FLASH_READ_ONFI_SIGNATURE_ADDRESS;
else
dma_buffer->data[0] = FLASH_READ_DEVICE_ID_ADDRESS;
dma_buffer->data[1] = EXTENDED_FETCH_ID | MSM_NAND_CMD_FETCH_ID;
dma_buffer->data[2] = 1;
dma_buffer->data[3] = 0xeeeeeeee;
dma_buffer->data[4] = 0xeeeeeeee;
cmd = dma_buffer->cmd;
msm_nand_prep_single_desc(cmd, MSM_NAND_ADDR0(info), WRITE,
dma_buffer->data[0], SPS_IOVEC_FLAG_LOCK);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_CMD(info), WRITE,
dma_buffer->data[1], 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE,
dma_buffer->data[2], SPS_IOVEC_FLAG_NWD);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_READ_ID(info), READ,
msm_virt_to_dma(chip, &dma_buffer->data[3]), 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_READ_ID2(info), READ,
msm_virt_to_dma(chip, &dma_buffer->data[4]),
SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_INT);
cmd++;
WARN_ON(cmd - dma_buffer->cmd > READID_CMDS);
dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd);
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
for (i = 0; i < dma_buffer->xfer.iovec_count; i++) {
iovec->addr = msm_virt_to_dma(chip, &dma_buffer->cmd[i].ce);
iovec->size = sizeof(struct sps_command_element);
iovec->flags = dma_buffer->cmd[i].flags;
iovec++;
}
mutex_lock(&info->lock);
err = msm_nand_get_device(chip->dev);
if (err)
goto out;
err = sps_transfer(info->sps.cmd_pipe.handle, &dma_buffer->xfer);
if (err) {
pr_err("Failed to submit commands %d\n", err);
msm_nand_put_device(chip->dev);
goto out;
}
err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d: (err:%d)\n",
(info->sps.cmd_pipe.index), err);
goto out;
}
pr_debug("Read ID register value 0x%x\n", dma_buffer->data[3]);
if (!read_onfi_signature)
pr_debug("nandid: %x maker %02x device %02x\n",
dma_buffer->data[3], dma_buffer->data[3] & 0xff,
(dma_buffer->data[3] >> 8) & 0xff);
*read_id = dma_buffer->data[3];
if (read_id2) {
pr_debug("Extended Read ID register value 0x%x\n",
dma_buffer->data[4]);
*read_id2 = dma_buffer->data[4];
}
err = msm_nand_put_device(chip->dev);
out:
mutex_unlock(&info->lock);
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
return err;
}
/*
* Contains data for common configuration registers that must be programmed
* for every NANDc operation.
*/
struct msm_nand_common_cfgs {
uint32_t cmd;
uint32_t addr0;
uint32_t addr1;
uint32_t cfg0;
uint32_t cfg1;
};
/*
* Function to prepare SPS command elements to write into NANDc configuration
* registers as per the data defined in struct msm_nand_common_cfgs. This is
* required for the following NANDc operations - Erase, Bad Block checking
* and for reading ONFI parameter page.
*/
static void msm_nand_prep_cfg_cmd_desc(struct msm_nand_info *info,
struct msm_nand_common_cfgs data,
struct msm_nand_sps_cmd **curr_cmd)
{
struct msm_nand_sps_cmd *cmd;
cmd = *curr_cmd;
msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_CMD(info), WRITE,
data.cmd, SPS_IOVEC_FLAG_LOCK);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_ADDR0(info), WRITE,
data.addr0, 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_ADDR1(info), WRITE,
data.addr1, 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_CFG0(info), WRITE,
data.cfg0, 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_CFG1(info), WRITE,
data.cfg1, 0);
cmd++;
*curr_cmd = cmd;
}
/*
* Function to check the CRC integrity check on ONFI parameter page read.
* For ONFI parameter page read, the controller ECC will be disabled. Hence,
* it is mandatory to manually compute CRC and check it against the value
* stored within ONFI page.
*/
static uint16_t msm_nand_flash_onfi_crc_check(uint8_t *buffer, uint16_t count)
{
int i;
uint16_t result;
for (i = 0; i < count; i++)
buffer[i] = bitrev8(buffer[i]);
result = bitrev16(crc16(bitrev16(0x4f4e), buffer, count));
for (i = 0; i < count; i++)
buffer[i] = bitrev8(buffer[i]);
return result;
}
/*
* Structure that contains NANDc register data for commands required
* for reading ONFI parameter page.
*/
struct msm_nand_flash_onfi_data {
struct msm_nand_common_cfgs cfg;
uint32_t exec;
uint32_t ecc_bch_cfg;
};
struct version {
uint16_t nand_major;
uint16_t nand_minor;
uint16_t qpic_major;
uint16_t qpic_minor;
};
static int msm_nand_version_check(struct msm_nand_info *info,
struct version *nandc_version)
{
uint32_t qpic_ver = 0, nand_ver = 0;
int err = 0;
/* Lookup the version to identify supported features */
err = msm_nand_flash_rd_reg(info, MSM_NAND_VERSION(info),
&nand_ver);
if (err) {
pr_err("Failed to read NAND_VERSION, err=%d\n", err);
goto out;
}
nandc_version->nand_major = (nand_ver & MSM_NAND_VERSION_MAJOR_MASK) >>
MSM_NAND_VERSION_MAJOR_SHIFT;
nandc_version->nand_minor = (nand_ver & MSM_NAND_VERSION_MINOR_MASK) >>
MSM_NAND_VERSION_MINOR_SHIFT;
err = msm_nand_flash_rd_reg(info, MSM_NAND_QPIC_VERSION(info),
&qpic_ver);
if (err) {
pr_err("Failed to read QPIC_VERSION, err=%d\n", err);
goto out;
}
nandc_version->qpic_major = (qpic_ver & MSM_NAND_VERSION_MAJOR_MASK) >>
MSM_NAND_VERSION_MAJOR_SHIFT;
nandc_version->qpic_minor = (qpic_ver & MSM_NAND_VERSION_MINOR_MASK) >>
MSM_NAND_VERSION_MINOR_SHIFT;
pr_info("nand_major:%d, nand_minor:%d, qpic_major:%d, qpic_minor:%d\n",
nandc_version->nand_major, nandc_version->nand_minor,
nandc_version->qpic_major, nandc_version->qpic_minor);
out:
return err;
}
/*
* Function to identify whether the attached NAND flash device is
* complaint to ONFI spec or not. If yes, then it reads the ONFI parameter
* page to get the device parameters.
*/
#define ONFI_CMDS 9
static int msm_nand_flash_onfi_probe(struct msm_nand_info *info)
{
struct msm_nand_chip *chip = &info->nand_chip;
struct flash_identification *flash = &info->flash_dev;
uint32_t crc_chk_count = 0, page_address = 0;
int ret = 0, i = 0, submitted_num_desc = 1;
/* SPS parameters */
struct msm_nand_sps_cmd *cmd, *curr_cmd;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
uint32_t rdata;
/* ONFI Identifier/Parameter Page parameters */
uint8_t *onfi_param_info_buf = NULL;
dma_addr_t dma_addr_param_info = 0;
struct onfi_param_page *onfi_param_page_ptr;
struct msm_nand_flash_onfi_data data;
uint32_t onfi_signature = 0;
/*
* The following 9 commands are required to get onfi parameters -
* flash, addr0, addr1, cfg0, cfg1, dev0_ecc_cfg,
* read_loc_0, exec, flash_status (read cmd).
*/
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[ONFI_CMDS];
struct msm_nand_sps_cmd cmd[ONFI_CMDS];
uint32_t flash_status;
} *dma_buffer;
/* Lookup the version to identify supported features */
struct version nandc_version = {0};
ret = msm_nand_version_check(info, &nandc_version);
if (!ret && !(nandc_version.nand_major == 1 &&
nandc_version.nand_minor >= 5 &&
nandc_version.qpic_major == 1 &&
nandc_version.qpic_minor >= 5)) {
ret = -EPERM;
goto out;
}
wait_event(chip->dma_wait_queue, (onfi_param_info_buf =
msm_nand_get_dma_buffer(chip, ONFI_PARAM_INFO_LENGTH)));
dma_addr_param_info = msm_virt_to_dma(chip, onfi_param_info_buf);
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer
(chip, sizeof(*dma_buffer))));
ret = msm_nand_flash_read_id(info, 1, &onfi_signature, NULL);
if (ret < 0) {
pr_err("Failed to read ONFI signature\n");
goto free_dma;
}
if (onfi_signature != ONFI_PARAMETER_PAGE_SIGNATURE) {
pr_info("Found a non ONFI device\n");
ret = -EIO;
goto free_dma;
}
memset(&data, 0, sizeof(struct msm_nand_flash_onfi_data));
/* Lookup the partition to which apps has access to */
for (i = 0; i < FLASH_PTABLE_MAX_PARTS_V4; i++) {
if (mtd_part[i].name && !strcmp("boot", mtd_part[i].name)) {
page_address = mtd_part[i].offset << 6;
break;
}
}
if (!page_address) {
pr_info("%s: no apps partition found in smem\n", __func__);
ret = -EPERM;
goto free_dma;
}
data.cfg.cmd = MSM_NAND_CMD_PAGE_READ_ONFI;
data.exec = 1;
data.cfg.addr0 = (page_address << 16) |
FLASH_READ_ONFI_PARAMETERS_ADDRESS;
data.cfg.addr1 = (page_address >> 16) & 0xFF;
data.cfg.cfg0 = MSM_NAND_CFG0_RAW_ONFI_PARAM_INFO;
data.cfg.cfg1 = MSM_NAND_CFG1_RAW_ONFI_PARAM_INFO;
data.ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
dma_buffer->flash_status = 0xeeeeeeee;
curr_cmd = cmd = dma_buffer->cmd;
msm_nand_prep_cfg_cmd_desc(info, data.cfg, &curr_cmd);
cmd = curr_cmd;
msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_ECC_CFG(info), WRITE,
data.ecc_bch_cfg, 0);
cmd++;
rdata = (0 << 0) | (ONFI_PARAM_INFO_LENGTH << 16) | (1 << 31);
msm_nand_prep_single_desc(cmd, MSM_NAND_READ_LOCATION_0(info), WRITE,
rdata, 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE,
data.exec, SPS_IOVEC_FLAG_NWD);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), READ,
msm_virt_to_dma(chip, &dma_buffer->flash_status),
SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_INT);
cmd++;
WARN_ON(cmd - dma_buffer->cmd > ONFI_CMDS);
dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd);
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
for (i = 0; i < dma_buffer->xfer.iovec_count; i++) {
iovec->addr = msm_virt_to_dma(chip,
&dma_buffer->cmd[i].ce);
iovec->size = sizeof(struct sps_command_element);
iovec->flags = dma_buffer->cmd[i].flags;
iovec++;
}
mutex_lock(&info->lock);
ret = msm_nand_get_device(chip->dev);
if (ret)
goto unlock_mutex;
/* Submit data descriptor */
ret = sps_transfer_one(info->sps.data_prod.handle, dma_addr_param_info,
ONFI_PARAM_INFO_LENGTH, NULL, SPS_IOVEC_FLAG_INT);
if (ret) {
pr_err("Failed to submit data descriptors %d\n", ret);
goto put_dev;
}
/* Submit command descriptors */
ret = sps_transfer(info->sps.cmd_pipe.handle,
&dma_buffer->xfer);
if (ret) {
pr_err("Failed to submit commands %d\n", ret);
goto put_dev;
}
ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count,
&iovec_temp);
if (ret) {
pr_err("Failed to get iovec for pipe %d: (ret:%d)\n",
(info->sps.cmd_pipe.index), ret);
goto put_dev;
}
ret = msm_nand_sps_get_iovec(info->sps.data_prod.handle,
info->sps.data_prod.index, submitted_num_desc,
&iovec_temp);
if (ret) {
pr_err("Failed to get iovec for pipe %d: (ret:%d)\n",
(info->sps.data_prod.index), ret);
goto put_dev;
}
ret = msm_nand_put_device(chip->dev);
mutex_unlock(&info->lock);
if (ret)
goto free_dma;
/* Check for flash status errors */
if (dma_buffer->flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
pr_err("MPU/OP err (0x%x) is set\n", dma_buffer->flash_status);
ret = -EIO;
goto free_dma;
}
for (crc_chk_count = 0; crc_chk_count < ONFI_PARAM_INFO_LENGTH
/ ONFI_PARAM_PAGE_LENGTH; crc_chk_count++) {
onfi_param_page_ptr =
(struct onfi_param_page *)
(&(onfi_param_info_buf
[ONFI_PARAM_PAGE_LENGTH *
crc_chk_count]));
if (msm_nand_flash_onfi_crc_check(
(uint8_t *)onfi_param_page_ptr,
ONFI_PARAM_PAGE_LENGTH - 2) ==
onfi_param_page_ptr->integrity_crc) {
break;
}
}
if (crc_chk_count >= ONFI_PARAM_INFO_LENGTH
/ ONFI_PARAM_PAGE_LENGTH) {
pr_err("CRC Check failed on param page\n");
ret = -EIO;
goto free_dma;
}
ret = msm_nand_flash_read_id(info, 0, &flash->flash_id, NULL);
if (ret < 0) {
pr_err("Failed to read flash ID\n");
goto free_dma;
}
flash->widebus = onfi_param_page_ptr->features_supported & 0x01;
flash->pagesize = onfi_param_page_ptr->number_of_data_bytes_per_page;
flash->blksize = onfi_param_page_ptr->number_of_pages_per_block *
flash->pagesize;
flash->oobsize = onfi_param_page_ptr->number_of_spare_bytes_per_page;
flash->density = onfi_param_page_ptr->number_of_blocks_per_logical_unit
* flash->blksize;
flash->ecc_correctability = onfi_param_page_ptr->
number_of_bits_ecc_correctability;
pr_info("Found an ONFI compliant device %s\n",
onfi_param_page_ptr->device_model);
/*
* Temporary hack for MT29F4G08ABC device.
* Since the device is not properly adhering
* to ONFi specification it is reporting
* as 16 bit device though it is 8 bit device!!!
*/
if (!strcmp(onfi_param_page_ptr->device_model, "MT29F4G08ABC"))
flash->widebus = 0;
goto free_dma;
put_dev:
msm_nand_put_device(chip->dev);
unlock_mutex:
mutex_unlock(&info->lock);
free_dma:
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
msm_nand_release_dma_buffer(chip, onfi_param_info_buf,
ONFI_PARAM_INFO_LENGTH);
out:
return ret;
}
/*
* Structure that contains read/write parameters required for reading/writing
* from/to a page.
*/
struct msm_nand_rw_params {
uint32_t page;
uint32_t page_count;
uint32_t sectordatasize;
uint32_t sectoroobsize;
uint32_t cwperpage;
uint32_t oob_len_cmd;
uint32_t oob_len_data;
uint32_t start_sector;
uint32_t oob_col;
dma_addr_t data_dma_addr;
dma_addr_t oob_dma_addr;
dma_addr_t ecc_dma_addr;
dma_addr_t data_dma_addr_curr;
dma_addr_t oob_dma_addr_curr;
dma_addr_t ecc_dma_addr_curr;
bool read;
};
/*
* Structure that contains NANDc register data required for reading/writing
* from/to a page.
*/
struct msm_nand_rw_reg_data {
uint32_t cmd;
uint32_t addr0;
uint32_t addr1;
uint32_t cfg0;
uint32_t cfg1;
uint32_t ecc_bch_cfg;
uint32_t exec;
uint32_t ecc_cfg;
uint32_t clrfstatus;
uint32_t clrrstatus;
};
/*
* Function that validates page read/write MTD parameters received from upper
* layers such as MTD/YAFFS2 and returns error for any unsupported operations
* by the driver. In case of success, it also maps the data and oob buffer
* received for DMA.
*/
static int msm_nand_validate_mtd_params(struct mtd_info *mtd, bool read,
loff_t offset,
struct mtd_oob_ops *ops,
struct msm_nand_rw_params *args)
{
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
int err = 0;
pr_debug("========================================================\n");
pr_debug("offset 0x%llx mode %d\ndatbuf 0x%pK datlen 0x%x\n",
offset, ops->mode, ops->datbuf, ops->len);
pr_debug("oobbuf 0x%pK ooblen 0x%x\n", ops->oobbuf, ops->ooblen);
if (ops->mode == MTD_OPS_PLACE_OOB) {
pr_err("MTD_OPS_PLACE_OOB is not supported\n");
err = -EINVAL;
goto out;
}
if (mtd->writesize == PAGE_SIZE_2K)
args->page = offset >> 11;
if (mtd->writesize == PAGE_SIZE_4K)
args->page = offset >> 12;
args->oob_len_cmd = ops->ooblen;
args->oob_len_data = ops->ooblen;
args->cwperpage = (mtd->writesize >> 9);
args->read = (read ? true : false);
if (offset & (mtd->writesize - 1)) {
pr_err("unsupported offset 0x%llx\n", offset);
err = -EINVAL;
goto out;
}
if (!read && !ops->datbuf) {
pr_err("No data buffer provided for write!!\n");
err = -EINVAL;
goto out;
}
if (ops->mode == MTD_OPS_RAW) {
if (!ops->datbuf) {
pr_err("No data buffer provided for RAW mode\n");
err = -EINVAL;
goto out;
} else if ((ops->len % (mtd->writesize +
mtd->oobsize)) != 0) {
pr_err("unsupported data len %d for RAW mode\n",
ops->len);
err = -EINVAL;
goto out;
}
args->page_count = ops->len / (mtd->writesize + mtd->oobsize);
} else if (ops->mode == MTD_OPS_AUTO_OOB) {
if (ops->datbuf && (ops->len % mtd->writesize) != 0) {
/* when ops->datbuf is NULL, ops->len can be ooblen */
pr_err("unsupported data len %d for AUTO mode\n",
ops->len);
err = -EINVAL;
goto out;
}
if (read && ops->oobbuf && !ops->datbuf) {
args->start_sector = args->cwperpage - 1;
args->page_count = ops->ooblen / mtd->oobavail;
if ((args->page_count == 0) && (ops->ooblen))
args->page_count = 1;
} else if (ops->datbuf) {
args->page_count = ops->len / mtd->writesize;
}
}
if (ops->datbuf) {
if (read)
memset(ops->datbuf, 0xFF, ops->len);
args->data_dma_addr_curr = args->data_dma_addr =
msm_nand_dma_map(chip->dev, ops->datbuf, ops->len,
(read ? DMA_FROM_DEVICE : DMA_TO_DEVICE));
if (dma_mapping_error(chip->dev, args->data_dma_addr)) {
pr_err("dma mapping failed for 0x%pK\n", ops->datbuf);
err = -EIO;
goto out;
}
}
if (ops->oobbuf) {
if (read)
memset(ops->oobbuf, 0xFF, ops->ooblen);
args->oob_dma_addr_curr = args->oob_dma_addr =
msm_nand_dma_map(chip->dev, ops->oobbuf, ops->ooblen,
(read ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE));
if (dma_mapping_error(chip->dev, args->oob_dma_addr)) {
pr_err("dma mapping failed for 0x%pK\n", ops->oobbuf);
err = -EIO;
goto dma_map_oobbuf_failed;
}
}
goto out;
dma_map_oobbuf_failed:
if (ops->datbuf)
dma_unmap_page(chip->dev, args->data_dma_addr, ops->len,
(read ? DMA_FROM_DEVICE : DMA_TO_DEVICE));
out:
return err;
}
/*
* Function that updates NANDc register data (struct msm_nand_rw_reg_data)
* required for page read/write.
*/
static void msm_nand_update_rw_reg_data(struct msm_nand_chip *chip,
struct mtd_oob_ops *ops,
struct msm_nand_rw_params *args,
struct msm_nand_rw_reg_data *data)
{
if (args->read) {
if (ops->mode != MTD_OPS_RAW) {
data->cmd = MSM_NAND_CMD_PAGE_READ_ECC;
data->cfg0 =
(chip->cfg0 & ~(7U << CW_PER_PAGE)) |
(((args->cwperpage-1) - args->start_sector)
<< CW_PER_PAGE);
data->cfg1 = chip->cfg1;
data->ecc_bch_cfg = chip->ecc_bch_cfg;
} else {
data->cmd = MSM_NAND_CMD_PAGE_READ_ALL;
data->cfg0 =
(chip->cfg0_raw & ~(7U << CW_PER_PAGE)) |
(((args->cwperpage-1) - args->start_sector)
<< CW_PER_PAGE);
data->cfg1 = chip->cfg1_raw;
data->ecc_bch_cfg = chip->ecc_cfg_raw;
}
} else {
if (ops->mode != MTD_OPS_RAW) {
data->cmd = MSM_NAND_CMD_PRG_PAGE;
data->cfg0 = chip->cfg0;
data->cfg1 = chip->cfg1;
data->ecc_bch_cfg = chip->ecc_bch_cfg;
} else {
data->cmd = MSM_NAND_CMD_PRG_PAGE_ALL;
data->cfg0 = chip->cfg0_raw;
data->cfg1 = chip->cfg1_raw;
data->ecc_bch_cfg = chip->ecc_cfg_raw;
}
data->clrfstatus = MSM_NAND_RESET_FLASH_STS;
data->clrrstatus = MSM_NAND_RESET_READ_STS;
}
data->exec = 1;
data->ecc_cfg = chip->ecc_buf_cfg;
}
/*
* Function to prepare series of SPS command descriptors required for a page
* read/write operation.
*/
static void msm_nand_prep_rw_cmd_desc(struct mtd_oob_ops *ops,
struct msm_nand_rw_params *args,
struct msm_nand_rw_reg_data *data,
struct msm_nand_info *info,
uint32_t curr_cw,
struct msm_nand_rw_cmd_desc *cmd_list,
uint32_t *cw_desc_cnt,
uint32_t ecc_parity_bytes)
{
struct msm_nand_chip *chip = &info->nand_chip;
uint32_t rdata;
/* read_location register parameters */
uint32_t offset, size, last_read;
struct sps_command_element *curr_ce, *start_ce;
uint32_t *flags_ptr, *num_ce_ptr;
if (curr_cw == args->start_sector) {
curr_ce = start_ce = &cmd_list->setup_desc.ce[0];
num_ce_ptr = &cmd_list->setup_desc.num_ce;
flags_ptr = &cmd_list->setup_desc.flags;
*flags_ptr = CMD_LCK;
cmd_list->count = 1;
msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_CMD(info), WRITE,
data->cmd);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_ADDR0(info), WRITE,
data->addr0);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_ADDR1(info), WRITE,
data->addr1);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_CFG0(info), WRITE,
data->cfg0);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_CFG1(info), WRITE,
data->cfg1);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_ECC_CFG(info), WRITE,
data->ecc_bch_cfg);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_EBI2_ECC_BUF_CFG(info),
WRITE, data->ecc_cfg);
curr_ce++;
if (!args->read) {
msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info),
WRITE, data->clrfstatus);
curr_ce++;
goto sub_exec_cmd;
} else {
msm_nand_prep_ce(curr_ce,
MSM_NAND_ERASED_CW_DETECT_CFG(info),
WRITE, CLR_ERASED_PAGE_DET);
curr_ce++;
msm_nand_prep_ce(curr_ce,
MSM_NAND_ERASED_CW_DETECT_CFG(info),
WRITE, SET_ERASED_PAGE_DET);
curr_ce++;
}
} else {
curr_ce = start_ce = &cmd_list->cw_desc[*cw_desc_cnt].ce[0];
num_ce_ptr = &cmd_list->cw_desc[*cw_desc_cnt].num_ce;
flags_ptr = &cmd_list->cw_desc[*cw_desc_cnt].flags;
*cw_desc_cnt += 1;
*flags_ptr = CMD;
cmd_list->count++;
}
if (!args->read)
goto sub_exec_cmd;
if (ops->mode == MTD_OPS_RAW) {
if (ecc_parity_bytes) {
rdata = (BYTES_517 << 0) | (ecc_parity_bytes << 16)
| (1 << 31);
msm_nand_prep_ce(curr_ce,
MSM_NAND_READ_LOCATION_0(info),
WRITE, rdata);
curr_ce++;
} else {
rdata = (0 << 0) | (chip->cw_size << 16) | (1 << 31);
msm_nand_prep_ce(curr_ce,
MSM_NAND_READ_LOCATION_0(info),
WRITE, rdata);
curr_ce++;
}
}
if (ops->mode == MTD_OPS_AUTO_OOB) {
if (ops->datbuf) {
offset = 0;
size = (curr_cw < (args->cwperpage - 1)) ? 516 :
(512 - ((args->cwperpage - 1) << 2));
last_read = (curr_cw < (args->cwperpage - 1)) ? 1 :
(ops->oobbuf ? 0 : 1);
rdata = (offset << 0) | (size << 16) |
(last_read << 31);
msm_nand_prep_ce(curr_ce,
MSM_NAND_READ_LOCATION_0(info),
WRITE,
rdata);
curr_ce++;
}
if (curr_cw == (args->cwperpage - 1) && ops->oobbuf) {
offset = 512 - ((args->cwperpage - 1) << 2);
size = (args->cwperpage) << 2;
if (size > args->oob_len_cmd)
size = args->oob_len_cmd;
args->oob_len_cmd -= size;
last_read = 1;
rdata = (offset << 0) | (size << 16) |
(last_read << 31);
if (!ops->datbuf)
msm_nand_prep_ce(curr_ce,
MSM_NAND_READ_LOCATION_0(info),
WRITE, rdata);
else
msm_nand_prep_ce(curr_ce,
MSM_NAND_READ_LOCATION_1(info),
WRITE, rdata);
curr_ce++;
}
}
sub_exec_cmd:
*flags_ptr |= NWD;
msm_nand_prep_ce(curr_ce, MSM_NAND_EXEC_CMD(info), WRITE, data->exec);
curr_ce++;
*num_ce_ptr = curr_ce - start_ce;
}
/*
* Function to prepare and submit SPS data descriptors required for a page
* read/write operation.
*/
static int msm_nand_submit_rw_data_desc(struct mtd_oob_ops *ops,
struct msm_nand_rw_params *args,
struct msm_nand_info *info,
uint32_t curr_cw,
uint32_t ecc_parity_bytes)
{
struct msm_nand_chip *chip = &info->nand_chip;
struct sps_pipe *data_pipe_handle;
uint32_t sectordatasize, sectoroobsize;
uint32_t sps_flags = 0;
int err = 0;
if (args->read)
data_pipe_handle = info->sps.data_prod.handle;
else
data_pipe_handle = info->sps.data_cons.handle;
if (ops->mode == MTD_OPS_RAW) {
if (ecc_parity_bytes && args->read) {
if (curr_cw == (args->cwperpage - 1))
sps_flags |= SPS_IOVEC_FLAG_INT;
/* read only ecc bytes */
err = sps_transfer_one(data_pipe_handle,
args->ecc_dma_addr_curr,
ecc_parity_bytes, NULL,
sps_flags);
if (err)
goto out;
args->ecc_dma_addr_curr += ecc_parity_bytes;
} else {
sectordatasize = chip->cw_size;
if (!args->read)
sps_flags = SPS_IOVEC_FLAG_EOT;
if (curr_cw == (args->cwperpage - 1))
sps_flags |= SPS_IOVEC_FLAG_INT;
err = sps_transfer_one(data_pipe_handle,
args->data_dma_addr_curr,
sectordatasize, NULL,
sps_flags);
if (err)
goto out;
args->data_dma_addr_curr += sectordatasize;
}
} else if (ops->mode == MTD_OPS_AUTO_OOB) {
if (ops->datbuf) {
sectordatasize = (curr_cw < (args->cwperpage - 1))
? 516 : (512 - ((args->cwperpage - 1) << 2));
if (!args->read) {
sps_flags = SPS_IOVEC_FLAG_EOT;
if (curr_cw == (args->cwperpage - 1) &&
ops->oobbuf)
sps_flags = 0;
}
if ((curr_cw == (args->cwperpage - 1)) && !ops->oobbuf)
sps_flags |= SPS_IOVEC_FLAG_INT;
err = sps_transfer_one(data_pipe_handle,
args->data_dma_addr_curr,
sectordatasize, NULL,
sps_flags);
if (err)
goto out;
args->data_dma_addr_curr += sectordatasize;
}
if (ops->oobbuf && (curr_cw == (args->cwperpage - 1))) {
sectoroobsize = args->cwperpage << 2;
if (sectoroobsize > args->oob_len_data)
sectoroobsize = args->oob_len_data;
if (!args->read)
sps_flags |= SPS_IOVEC_FLAG_EOT;
sps_flags |= SPS_IOVEC_FLAG_INT;
err = sps_transfer_one(data_pipe_handle,
args->oob_dma_addr_curr,
sectoroobsize, NULL,
sps_flags);
if (err)
goto out;
args->oob_dma_addr_curr += sectoroobsize;
args->oob_len_data -= sectoroobsize;
}
}
out:
return err;
}
/*
* Read ECC bytes and check whether page is erased or not.
*
* The NAND devices manufactured with newer process node technology are
* susceptible to bit-flips. These bit-flips are easily fixable with the
* ECC engine and ECC information stored on the NAND device. This device
* specific information is found in the data sheet for the NAND device
* and is usually specified as a "number of bit-flips expected per code-
* word". For example, "a single bit-flip per codeword". Also this means
* that the number of ECC errors don't increase over period of time as in
* the past and can't be used to predict a "bad-block about to happen"
* situation anymore.
*
* So what this means to erased pages:
* Since ECC data for an erased page is all 0xFF's, the ECC engine would
* not be able to correct any bit-flips that occur in these newer parts.
* If the NAND controller is unable to identify the erased page due to
* the bit-flips, then there would be "uncorrectable ECC errors" detected
* and would get reported to file system layer (YAFFS2/UBIFS etc) and would
* result in a good block being marked as a bad block and also lead to
* error scenarios.
* So to handle this, the following will be done by software until newer
* NAND controller hardware is avialable that can detected erased pages
* with bit-flips successfully.
*
* 1. msm_nand_read_oob() calls this function when "uncorrectable ECC
* errors" occur.
* 2. This function then performs a raw read of the page.
* 3. This read is done to extract ECC bytes and not data from that page.
* 4. For each codeword’s ECC data, the following is done
* a. Count number of zero bits
* b. If that count is greater than <BIT-FLIPS-EXPECTED>, then it is
* not an erased page.
* c. Else repeat for next codeword’s ECC data
* d. If all codewords have less than <BIT-FLIPS-EXPECTED> bits of
* zeros, then it’s considered an erased page.
*
* Since "uncorrectable ECC errors" do not occur except for either an
* erased page or in the case of an actual errror, this solution would
* work.
*
*/
static int msm_nand_is_erased_page(struct mtd_info *mtd, loff_t from,
struct mtd_oob_ops *ops,
struct msm_nand_rw_params *rw_params,
bool *erased_page)
{
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
uint32_t cwperpage = (mtd->writesize >> 9);
int err, submitted_num_desc = 0;
uint32_t n = 0, num_zero_bits = 0, total_ecc_byte_cnt;
struct msm_nand_rw_reg_data data;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
struct mtd_oob_ops raw_ops;
/*
* The following 6 commands will be sent only once for the first
* codeword (CW) - addr0, addr1, dev0_cfg0, dev0_cfg1,
* dev0_ecc_cfg, ebi2_ecc_buf_cfg. The following 6 commands will
* be sent for every CW - flash, read_location_0, read_location_1,
* exec, flash_status and buffer_status.
*/
struct msm_nand_rw_cmd_desc *cmd_list = NULL;
uint32_t cw_desc_cnt = 0;
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[MAX_DESC];
struct {
uint32_t count;
struct msm_nand_cmd_setup_desc setup_desc;
struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC - 1];
} cmd_list;
struct {
uint32_t flash_status;
uint32_t buffer_status;
uint32_t erased_cw_status;
} result[MAX_CW_PER_PAGE];
} *dma_buffer;
uint8_t *ecc;
pr_debug("========================================================\n");
total_ecc_byte_cnt = (chip->ecc_parity_bytes * cwperpage);
memcpy(&raw_ops, ops, sizeof(struct mtd_oob_ops));
raw_ops.mode = MTD_OPS_RAW;
ecc = kzalloc(total_ecc_byte_cnt, GFP_KERNEL);
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer(
chip, sizeof(*dma_buffer))));
memset(&data, 0, sizeof(struct msm_nand_rw_reg_data));
msm_nand_update_rw_reg_data(chip, &raw_ops, rw_params, &data);
cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list;
/* map the ecc for dma operations */
rw_params->ecc_dma_addr_curr = rw_params->ecc_dma_addr =
dma_map_single(chip->dev, ecc, total_ecc_byte_cnt,
DMA_FROM_DEVICE);
data.addr0 = (rw_params->page << 16) | rw_params->oob_col;
data.addr1 = (rw_params->page >> 16) & 0xff;
for (n = rw_params->start_sector; n < cwperpage; n++) {
struct sps_command_element *curr_ce, *start_ce;
dma_buffer->result[n].flash_status = 0xeeeeeeee;
dma_buffer->result[n].buffer_status = 0xeeeeeeee;
dma_buffer->result[n].erased_cw_status = 0xeeeeee00;
msm_nand_prep_rw_cmd_desc(&raw_ops, rw_params, &data, info,
n, cmd_list, &cw_desc_cnt,
chip->ecc_parity_bytes);
start_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0];
curr_ce = start_ce;
cmd_list->cw_desc[cw_desc_cnt].flags = CMD;
if (n == (cwperpage - 1))
cmd_list->cw_desc[cw_desc_cnt].flags |=
INT_UNLCK;
cmd_list->count++;
msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->result[n].flash_status));
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_BUFFER_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->result[n].buffer_status));
curr_ce++;
msm_nand_prep_ce(curr_ce,
MSM_NAND_ERASED_CW_DETECT_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->result[n].erased_cw_status));
curr_ce++;
cmd_list->cw_desc[cw_desc_cnt++].num_ce = curr_ce -
start_ce;
}
dma_buffer->xfer.iovec_count = cmd_list->count;
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
iovec->addr = msm_virt_to_dma(chip,
&cmd_list->setup_desc.ce[0]);
iovec->size = sizeof(struct sps_command_element) *
cmd_list->setup_desc.num_ce;
iovec->flags = cmd_list->setup_desc.flags;
iovec++;
for (n = 0; n < (cmd_list->count - 1); n++) {
iovec->addr = msm_virt_to_dma(chip,
&cmd_list->cw_desc[n].ce[0]);
iovec->size = sizeof(struct sps_command_element) *
cmd_list->cw_desc[n].num_ce;
iovec->flags = cmd_list->cw_desc[n].flags;
iovec++;
}
mutex_lock(&info->lock);
err = msm_nand_get_device(chip->dev);
if (err)
goto unlock_mutex;
/* Submit data descriptors */
for (n = rw_params->start_sector; n < cwperpage; n++) {
err = msm_nand_submit_rw_data_desc(&raw_ops,
rw_params, info, n,
chip->ecc_parity_bytes);
if (err) {
pr_err("Failed to submit data descs %d\n", err);
panic("error in nand driver\n");
goto put_dev;
}
}
submitted_num_desc = cwperpage - rw_params->start_sector;
/* Submit command descriptors */
err = sps_transfer(info->sps.cmd_pipe.handle,
&dma_buffer->xfer);
if (err) {
pr_err("Failed to submit commands %d\n", err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index,
dma_buffer->xfer.iovec_count,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d: (err:%d)\n",
(info->sps.cmd_pipe.index), err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.data_prod.handle,
info->sps.data_prod.index, submitted_num_desc,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d: (err:%d)\n",
(info->sps.data_prod.index), err);
goto put_dev;
}
err = msm_nand_put_device(chip->dev);
mutex_unlock(&info->lock);
if (err)
goto free_dma;
pr_debug("addr0: 0x%08x, addr1: 0x%08x\n", data.addr0, data.addr1);
for (n = rw_params->start_sector; n < cwperpage; n++)
pr_debug("cw %d: flash_sts %x buffr_sts %x, erased_cw_status: %x\n",
n, dma_buffer->result[n].flash_status,
dma_buffer->result[n].buffer_status,
dma_buffer->result[n].erased_cw_status);
goto free_dma;
put_dev:
msm_nand_put_device(chip->dev);
unlock_mutex:
mutex_unlock(&info->lock);
free_dma:
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
/* umap ecc dma memory */
dma_unmap_single(chip->dev, rw_params->ecc_dma_addr,
total_ecc_byte_cnt, DMA_FROM_DEVICE);
/* check for bit flips in ecc data */
for (n = rw_params->start_sector; n < cwperpage; n++) {
uint8_t *ecc_temp = ecc;
int last_pos = 0, next_pos = 0;
int ecc_bytes_percw_in_bits = (chip->ecc_parity_bytes * 8);
do {
last_pos = find_next_zero_bit(ecc_temp,
ecc_bytes_percw_in_bits, next_pos);
if (last_pos < ecc_bytes_percw_in_bits)
num_zero_bits++;
if (num_zero_bits > 4) {
*erased_page = false;
goto free_mem;
}
next_pos = last_pos + 1;
} while (last_pos < ecc_bytes_percw_in_bits);
num_zero_bits = last_pos = next_pos = 0;
ecc_temp += chip->ecc_parity_bytes;
}
if ((n == cwperpage) && (num_zero_bits <= 4))
*erased_page = true;
free_mem:
kfree(ecc);
pr_debug("========================================================\n");
return err;
}
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to read a
* page with main or/and spare data.
*/
static int msm_nand_read_oob(struct mtd_info *mtd, loff_t from,
struct mtd_oob_ops *ops)
{
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
struct flash_identification *flash_dev = &info->flash_dev;
uint32_t cwperpage = (mtd->writesize >> 9);
int err, pageerr = 0, rawerr = 0, submitted_num_desc = 0;
uint32_t n = 0, pages_read = 0;
uint32_t ecc_errors = 0, total_ecc_errors = 0, ecc_capability;
struct msm_nand_rw_params rw_params;
struct msm_nand_rw_reg_data data;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
bool erased_page;
uint64_t fix_data_in_pages = 0;
ktime_t start;
/*
* The following 6 commands will be sent only once for the first
* codeword (CW) - addr0, addr1, dev0_cfg0, dev0_cfg1,
* dev0_ecc_cfg, ebi2_ecc_buf_cfg. The following 6 commands will
* be sent for every CW - flash, read_location_0, read_location_1,
* exec, flash_status and buffer_status.
*/
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[MAX_DESC];
struct {
uint32_t count;
struct msm_nand_cmd_setup_desc setup_desc;
struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC - 1];
} cmd_list;
struct {
uint32_t flash_status;
uint32_t buffer_status;
uint32_t erased_cw_status;
} result[MAX_CW_PER_PAGE];
} *dma_buffer;
struct msm_nand_rw_cmd_desc *cmd_list = NULL;
if (unlikely(enable_perfstats))
start = ktime_get();
memset(&rw_params, 0, sizeof(struct msm_nand_rw_params));
err = msm_nand_validate_mtd_params(mtd, true, from, ops, &rw_params);
if (err)
goto validate_mtd_params_failed;
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer(
chip, sizeof(*dma_buffer))));
rw_params.oob_col = rw_params.start_sector * chip->cw_size;
if (chip->cfg1 & (1 << WIDE_FLASH))
rw_params.oob_col >>= 1;
memset(&data, 0, sizeof(struct msm_nand_rw_reg_data));
msm_nand_update_rw_reg_data(chip, ops, &rw_params, &data);
cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list;
ecc_capability = flash_dev->ecc_capability;
while (rw_params.page_count-- > 0) {
uint32_t cw_desc_cnt = 0;
erased_page = false;
data.addr0 = (rw_params.page << 16) | rw_params.oob_col;
data.addr1 = (rw_params.page >> 16) & 0xff;
for (n = rw_params.start_sector; n < cwperpage; n++) {
struct sps_command_element *curr_ce, *start_ce;
dma_buffer->result[n].flash_status = 0xeeeeeeee;
dma_buffer->result[n].buffer_status = 0xeeeeeeee;
dma_buffer->result[n].erased_cw_status = 0xeeeeee00;
msm_nand_prep_rw_cmd_desc(ops, &rw_params, &data, info,
n, cmd_list, &cw_desc_cnt, 0);
start_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0];
curr_ce = start_ce;
cmd_list->cw_desc[cw_desc_cnt].flags = CMD;
if (n == (cwperpage - 1))
cmd_list->cw_desc[cw_desc_cnt].flags |=
INT_UNLCK;
cmd_list->count++;
msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->result[n].flash_status));
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_BUFFER_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->result[n].buffer_status));
curr_ce++;
msm_nand_prep_ce(curr_ce,
MSM_NAND_ERASED_CW_DETECT_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->result[n].erased_cw_status));
curr_ce++;
cmd_list->cw_desc[cw_desc_cnt++].num_ce = curr_ce -
start_ce;
}
dma_buffer->xfer.iovec_count = cmd_list->count;
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
iovec->addr = msm_virt_to_dma(chip,
&cmd_list->setup_desc.ce[0]);
iovec->size = sizeof(struct sps_command_element) *
cmd_list->setup_desc.num_ce;
iovec->flags = cmd_list->setup_desc.flags;
iovec++;
for (n = 0; n < (cmd_list->count - 1); n++) {
iovec->addr = msm_virt_to_dma(chip,
&cmd_list->cw_desc[n].ce[0]);
iovec->size = sizeof(struct sps_command_element) *
cmd_list->cw_desc[n].num_ce;
iovec->flags = cmd_list->cw_desc[n].flags;
iovec++;
}
mutex_lock(&info->lock);
err = msm_nand_get_device(chip->dev);
if (err)
goto unlock_mutex;
/* Submit data descriptors */
for (n = rw_params.start_sector; n < cwperpage; n++) {
err = msm_nand_submit_rw_data_desc(ops,
&rw_params, info, n, 0);
if (err) {
pr_err("Failed to submit data descs %d\n", err);
panic("error in nand driver\n");
goto put_dev;
}
}
if (ops->mode == MTD_OPS_RAW) {
submitted_num_desc = cwperpage - rw_params.start_sector;
} else if (ops->mode == MTD_OPS_AUTO_OOB) {
if (ops->datbuf)
submitted_num_desc = cwperpage -
rw_params.start_sector;
if (ops->oobbuf)
submitted_num_desc++;
}
/* Submit command descriptors */
err = sps_transfer(info->sps.cmd_pipe.handle,
&dma_buffer->xfer);
if (err) {
pr_err("Failed to submit commands %d\n", err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index,
dma_buffer->xfer.iovec_count,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d: (err: %d)\n",
(info->sps.cmd_pipe.index), err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.data_prod.handle,
info->sps.data_prod.index, submitted_num_desc,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d: (err: %d)\n",
(info->sps.data_prod.index), err);
goto put_dev;
}
err = msm_nand_put_device(chip->dev);
mutex_unlock(&info->lock);
if (err)
goto free_dma;
/* Check for flash status errors */
pageerr = rawerr = 0;
for (n = rw_params.start_sector; n < cwperpage; n++) {
if (dma_buffer->result[n].flash_status & (FS_OP_ERR |
FS_MPU_ERR)) {
rawerr = -EIO;
/*
* Check if ECC error was due to an erased
* codeword. If so, ignore the error.
*
* NOTE: There is a bug in erased page
* detection hardware block when reading
* only spare data. In order to work around
* this issue, instead of using PAGE_ALL_ERASED
* bit to check for whether a whole page is
* erased or not, we use CODEWORD_ALL_ERASED
* and CODEWORD_ERASED bits together and check
* each codeword that has FP_OP_ERR bit set is
* an erased codeword or not.
*/
if ((dma_buffer->result[n].erased_cw_status &
ERASED_CW) == ERASED_CW) {
/*
* At least one code word is detected
* as an erased code word.
*/
pr_debug("erased codeword detected - ignore ecc error\n");
continue;
}
pageerr = rawerr;
break;
}
}
/* check for uncorrectable errors */
if (pageerr) {
for (n = rw_params.start_sector; n < cwperpage; n++) {
if (dma_buffer->result[n].buffer_status &
BS_UNCORRECTABLE_BIT) {
/*
* Check if page is actually
* erased or not.
*/
err = msm_nand_is_erased_page(mtd,
from, ops,
&rw_params,
&erased_page);
if (err)
goto free_dma;
if (!erased_page) {
mtd->ecc_stats.failed++;
pageerr = -EBADMSG;
break;
}
pageerr = 0;
pr_debug("Uncorrectable ECC errors dectected on an erased page and has been fixed.\n");
break;
}
}
}
if (rawerr && !pageerr && erased_page) {
/*
* This means an erased page had bit flips and now
* those bit-flips need to be cleared in the data
* being sent to upper layers. This will keep track
* of those pages and at the end, the data will be
* fixed before this function returns.
* Note that a whole page worth of data will be fixed
* and this will only handle about 64 pages being read
* at a time i.e. one erase block worth of pages.
*/
fix_data_in_pages |= BIT(rw_params.page_count);
}
/* check for correctable errors */
if (!rawerr) {
for (n = rw_params.start_sector; n < cwperpage; n++) {
ecc_errors =
dma_buffer->result[n].buffer_status
& BS_CORRECTABLE_ERR_MSK;
if (ecc_errors) {
total_ecc_errors += ecc_errors;
mtd->ecc_stats.corrected += ecc_errors;
/*
* Since the nand device can have the
* ecc errors even on the first ever
* write. Any reporting of EUCLEAN
* when there are less then the ecc
* capability of the device is not
* useful.
*
* Also don't report EUCLEAN unless
* the enable_euclean is set.
*/
if (enable_euclean &&
ecc_errors >= ecc_capability)
pageerr = -EUCLEAN;
}
}
}
if (pageerr && (pageerr != -EUCLEAN || err == 0))
err = pageerr;
if (rawerr && !pageerr) {
pr_debug("%llx %x %x empty page\n",
(loff_t)rw_params.page * mtd->writesize,
ops->len, ops->ooblen);
} else {
for (n = rw_params.start_sector; n < cwperpage; n++)
pr_debug("cw %d: flash_sts %x buffr_sts %x, erased_cw_status: %x, pageerr: %d, rawerr: %d\n",
n, dma_buffer->result[n].flash_status,
dma_buffer->result[n].buffer_status,
dma_buffer->result[n].erased_cw_status,
pageerr, rawerr);
}
if (err && err != -EUCLEAN && err != -EBADMSG)
goto free_dma;
pages_read++;
rw_params.page++;
}
goto free_dma;
put_dev:
msm_nand_put_device(chip->dev);
unlock_mutex:
mutex_unlock(&info->lock);
free_dma:
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
if (ops->oobbuf)
dma_unmap_page(chip->dev, rw_params.oob_dma_addr,
ops->ooblen, DMA_FROM_DEVICE);
if (ops->datbuf)
dma_unmap_page(chip->dev, rw_params.data_dma_addr,
ops->len, DMA_BIDIRECTIONAL);
/*
* If there were any erased pages detected with ECC errors, then
* it is most likely that the data is not all 0xff. So memset that
* page to all 0xff.
*/
while (fix_data_in_pages) {
int temp_page = 0, oobsize = rw_params.cwperpage << 2;
int count = 0, offset = 0;
temp_page = fix_data_in_pages & BIT_MASK(0);
fix_data_in_pages = fix_data_in_pages >> 1;
count++;
if (!temp_page)
continue;
offset = (count - 1) * mtd->writesize;
if (ops->datbuf)
memset((ops->datbuf + offset), 0xff, mtd->writesize);
offset = (count - 1) * oobsize;
if (ops->oobbuf)
memset(ops->oobbuf + offset, 0xff, oobsize);
}
validate_mtd_params_failed:
if (ops->mode != MTD_OPS_RAW)
ops->retlen = mtd->writesize * pages_read;
else
ops->retlen = (mtd->writesize + mtd->oobsize) * pages_read;
ops->oobretlen = ops->ooblen - rw_params.oob_len_data;
if (err)
pr_err("0x%llx datalen 0x%x ooblen %x err %d corrected %d\n",
from, ops->datbuf ? ops->len : 0, ops->ooblen, err,
total_ecc_errors);
pr_debug("ret %d, retlen %d oobretlen %d\n",
err, ops->retlen, ops->oobretlen);
pr_debug("========================================================\n");
if (unlikely(enable_perfstats) && likely(!err))
msm_nand_update_read_perf_stats(info, start, ops->retlen);
return err;
}
/**
* msm_nand_read_partial_page() - read partial page
* @mtd: pointer to mtd info
* @from: start address of the page
* @ops: pointer to mtd_oob_ops
*
* Reads a page into a bounce buffer and copies the required
* number of bytes to actual buffer. The pages that are aligned
* do not use bounce buffer.
*/
static int msm_nand_read_partial_page(struct mtd_info *mtd,
loff_t from, struct mtd_oob_ops *ops)
{
int err = 0;
unsigned char *actual_buf;
unsigned char *bounce_buf;
loff_t aligned_from;
loff_t offset;
size_t len;
size_t actual_len, ret_len;
int is_euclean = 0;
int is_ebadmsg = 0;
actual_len = ops->len;
ret_len = 0;
actual_buf = ops->datbuf;
bounce_buf = kmalloc(mtd->writesize, GFP_KERNEL);
if (!bounce_buf) {
err = -ENOMEM;
goto out;
}
/* Get start address of page to read from */
ops->len = mtd->writesize;
offset = from & (mtd->writesize - 1);
aligned_from = from - offset;
for (;;) {
bool no_copy = false;
len = mtd->writesize - offset;
if (len > actual_len)
len = actual_len;
if (offset == 0 && len == mtd->writesize)
no_copy = true;
if (!virt_addr_valid(actual_buf) &&
!is_buffer_in_page(actual_buf, ops->len))
no_copy = false;
ops->datbuf = no_copy ? actual_buf : bounce_buf;
err = msm_nand_read_oob(mtd, aligned_from, ops);
if (err == -EUCLEAN) {
is_euclean = 1;
err = 0;
}
if (err == -EBADMSG) {
is_ebadmsg = 1;
err = 0;
}
if (err < 0) {
/* Clear previously set EUCLEAN / EBADMSG */
is_euclean = 0;
is_ebadmsg = 0;
ret_len = ops->retlen;
break;
}
if (!no_copy)
memcpy(actual_buf, bounce_buf + offset, len);
actual_len -= len;
ret_len += len;
if (actual_len == 0)
break;
actual_buf += len;
offset = 0;
aligned_from += mtd->writesize;
}
ops->retlen = ret_len;
kfree(bounce_buf);
out:
if (is_euclean == 1)
err = -EUCLEAN;
/* Snub EUCLEAN if we also have EBADMSG */
if (is_ebadmsg == 1)
err = -EBADMSG;
return err;
}
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to read a
* page with only main data.
*/
static int msm_nand_read(struct mtd_info *mtd, loff_t from, size_t len,
size_t *retlen, u_char *buf)
{
int ret;
int is_euclean = 0;
int is_ebadmsg = 0;
struct mtd_oob_ops ops;
unsigned char *bounce_buf = NULL;
ops.mode = MTD_OPS_AUTO_OOB;
ops.retlen = 0;
ops.ooblen = 0;
ops.oobbuf = NULL;
*retlen = 0;
if (!(from & (mtd->writesize - 1)) && !(len % mtd->writesize)) {
/*
* Handle reading of large size read buffer in vmalloc
* address space that does not fit in an MMU page.
*/
if (!virt_addr_valid(buf) && !is_buffer_in_page(buf, len)) {
ops.len = mtd->writesize;
bounce_buf = kmalloc(ops.len, GFP_KERNEL);
if (!bounce_buf) {
ret = -ENOMEM;
goto out;
}
for (;;) {
bool no_copy = false;
if (!is_buffer_in_page(buf, ops.len)) {
memcpy(bounce_buf, buf, ops.len);
ops.datbuf = (uint8_t *) bounce_buf;
} else {
ops.datbuf = (uint8_t *) buf;
no_copy = true;
}
ret = msm_nand_read_oob(mtd, from, &ops);
if (ret == -EUCLEAN) {
is_euclean = 1;
ret = 0;
}
if (ret == -EBADMSG) {
is_ebadmsg = 1;
ret = 0;
}
if (ret < 0) {
/* Clear previously set errors */
is_euclean = 0;
is_ebadmsg = 0;
break;
}
if (!no_copy)
memcpy(buf, bounce_buf, ops.retlen);
len -= ops.retlen;
*retlen += ops.retlen;
if (len == 0)
break;
buf += ops.retlen;
from += ops.retlen;
if (len < mtd->writesize) {
ops.len = len;
ops.datbuf = buf;
ret = msm_nand_read_partial_page(
mtd, from, &ops);
*retlen += ops.retlen;
break;
}
}
kfree(bounce_buf);
} else {
ops.len = len;
ops.datbuf = (uint8_t *)buf;
ret = msm_nand_read_oob(mtd, from, &ops);
*retlen = ops.retlen;
}
} else {
ops.len = len;
ops.datbuf = (uint8_t *)buf;
ret = msm_nand_read_partial_page(mtd, from, &ops);
*retlen = ops.retlen;
}
out:
if (is_euclean == 1)
ret = -EUCLEAN;
/* Snub EUCLEAN if we also have EBADMSG */
if (is_ebadmsg == 1)
ret = -EBADMSG;
return ret;
}
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to write a
* page with both main and spare data.
*/
static int msm_nand_write_oob(struct mtd_info *mtd, loff_t to,
struct mtd_oob_ops *ops)
{
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
uint32_t cwperpage = (mtd->writesize >> 9);
uint32_t n, flash_sts, pages_written = 0;
int err = 0, submitted_num_desc = 0;
struct msm_nand_rw_params rw_params;
struct msm_nand_rw_reg_data data;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
ktime_t start;
/*
* The following 7 commands will be sent only once :
* For first codeword (CW) - addr0, addr1, dev0_cfg0, dev0_cfg1,
* dev0_ecc_cfg, ebi2_ecc_buf_cfg.
* For last codeword (CW) - read_status(write)
*
* The following 4 commands will be sent for every CW :
* flash, exec, flash_status (read), flash_status (write).
*/
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[MAX_DESC + 1];
struct {
uint32_t count;
struct msm_nand_cmd_setup_desc setup_desc;
struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC];
} cmd_list;
struct {
uint32_t flash_status;
} data[MAX_CW_PER_PAGE];
} *dma_buffer;
struct msm_nand_rw_cmd_desc *cmd_list = NULL;
if (unlikely(enable_perfstats))
start = ktime_get();
memset(&rw_params, 0, sizeof(struct msm_nand_rw_params));
err = msm_nand_validate_mtd_params(mtd, false, to, ops, &rw_params);
if (err)
goto validate_mtd_params_failed;
wait_event(chip->dma_wait_queue, (dma_buffer =
msm_nand_get_dma_buffer(chip, sizeof(*dma_buffer))));
memset(&data, 0, sizeof(struct msm_nand_rw_reg_data));
msm_nand_update_rw_reg_data(chip, ops, &rw_params, &data);
cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list;
while (rw_params.page_count-- > 0) {
uint32_t cw_desc_cnt = 0;
struct sps_command_element *curr_ce, *start_ce;
data.addr0 = (rw_params.page << 16);
data.addr1 = (rw_params.page >> 16) & 0xff;
for (n = 0; n < cwperpage ; n++) {
dma_buffer->data[n].flash_status = 0xeeeeeeee;
msm_nand_prep_rw_cmd_desc(ops, &rw_params, &data, info,
n, cmd_list, &cw_desc_cnt, 0);
curr_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0];
cmd_list->cw_desc[cw_desc_cnt].flags = CMD;
cmd_list->count++;
msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info),
READ, msm_virt_to_dma(chip,
&dma_buffer->data[n].flash_status));
cmd_list->cw_desc[cw_desc_cnt++].num_ce = 1;
}
start_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0];
curr_ce = start_ce;
cmd_list->cw_desc[cw_desc_cnt].flags = CMD_INT_UNLCK;
cmd_list->count++;
msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info),
WRITE, data.clrfstatus);
curr_ce++;
msm_nand_prep_ce(curr_ce, MSM_NAND_READ_STATUS(info),
WRITE, data.clrrstatus);
curr_ce++;
cmd_list->cw_desc[cw_desc_cnt++].num_ce = curr_ce - start_ce;
dma_buffer->xfer.iovec_count = cmd_list->count;
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
iovec->addr = msm_virt_to_dma(chip,
&cmd_list->setup_desc.ce[0]);
iovec->size = sizeof(struct sps_command_element) *
cmd_list->setup_desc.num_ce;
iovec->flags = cmd_list->setup_desc.flags;
iovec++;
for (n = 0; n < (cmd_list->count - 1); n++) {
iovec->addr = msm_virt_to_dma(chip,
&cmd_list->cw_desc[n].ce[0]);
iovec->size = sizeof(struct sps_command_element) *
cmd_list->cw_desc[n].num_ce;
iovec->flags = cmd_list->cw_desc[n].flags;
iovec++;
}
mutex_lock(&info->lock);
err = msm_nand_get_device(chip->dev);
if (err)
goto unlock_mutex;
/* Submit data descriptors */
for (n = 0; n < cwperpage; n++) {
err = msm_nand_submit_rw_data_desc(ops,
&rw_params, info, n, 0);
if (err) {
pr_err("Failed to submit data descs %d\n", err);
panic("Error in nand driver\n");
goto put_dev;
}
}
if (ops->mode == MTD_OPS_RAW) {
submitted_num_desc = n;
} else if (ops->mode == MTD_OPS_AUTO_OOB) {
if (ops->datbuf)
submitted_num_desc = n;
if (ops->oobbuf)
submitted_num_desc++;
}
/* Submit command descriptors */
err = sps_transfer(info->sps.cmd_pipe.handle,
&dma_buffer->xfer);
if (err) {
pr_err("Failed to submit commands %d\n", err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index,
dma_buffer->xfer.iovec_count,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d (err:%d)\n",
(info->sps.cmd_pipe.index), err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.data_cons.handle,
info->sps.data_cons.index, submitted_num_desc,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d (err:%d)\n",
(info->sps.data_cons.index), err);
goto put_dev;
}
err = msm_nand_put_device(chip->dev);
mutex_unlock(&info->lock);
if (err)
goto free_dma;
for (n = 0; n < cwperpage; n++)
pr_debug("write pg %d: flash_status[%d] = %x\n",
rw_params.page, n,
dma_buffer->data[n].flash_status);
/* Check for flash status errors */
for (n = 0; n < cwperpage; n++) {
flash_sts = dma_buffer->data[n].flash_status;
if (flash_sts & (FS_OP_ERR | FS_MPU_ERR)) {
pr_err("MPU/OP err (0x%x) set\n", flash_sts);
err = -EIO;
goto free_dma;
}
if (n == (cwperpage - 1)) {
if (!(flash_sts & FS_DEVICE_WP) ||
(flash_sts & FS_DEVICE_STS_ERR)) {
pr_err("Dev sts err 0x%x\n", flash_sts);
err = -EIO;
goto free_dma;
}
}
}
pages_written++;
rw_params.page++;
}
goto free_dma;
put_dev:
msm_nand_put_device(chip->dev);
unlock_mutex:
mutex_unlock(&info->lock);
free_dma:
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
if (ops->oobbuf)
dma_unmap_page(chip->dev, rw_params.oob_dma_addr,
ops->ooblen, DMA_TO_DEVICE);
if (ops->datbuf)
dma_unmap_page(chip->dev, rw_params.data_dma_addr,
ops->len, DMA_TO_DEVICE);
validate_mtd_params_failed:
if (ops->mode != MTD_OPS_RAW)
ops->retlen = mtd->writesize * pages_written;
else
ops->retlen = (mtd->writesize + mtd->oobsize) * pages_written;
ops->oobretlen = ops->ooblen - rw_params.oob_len_data;
if (err)
pr_err("to %llx datalen %x ooblen %x failed with err %d\n",
to, ops->len, ops->ooblen, err);
pr_debug("ret %d, retlen %d oobretlen %d\n",
err, ops->retlen, ops->oobretlen);
pr_debug("================================================\n");
if (unlikely(enable_perfstats) && likely(!err))
msm_nand_update_write_perf_stats(info, start, ops->retlen);
return err;
}
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to write a
* page with only main data.
*/
static int msm_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
size_t *retlen, const u_char *buf)
{
int ret;
struct mtd_oob_ops ops;
unsigned char *bounce_buf = NULL;
ops.mode = MTD_OPS_AUTO_OOB;
ops.retlen = 0;
ops.ooblen = 0;
ops.oobbuf = NULL;
/* partial page writes are not supported */
if ((to & (mtd->writesize - 1)) || (len % mtd->writesize)) {
ret = -EINVAL;
*retlen = ops.retlen;
pr_err("%s: partial page writes are not supported\n", __func__);
goto out;
}
/*
* Handle writing of large size write buffer in vmalloc
* address space that does not fit in an MMU page.
*/
if (!virt_addr_valid(buf) && !is_buffer_in_page(buf, len)) {
ops.len = mtd->writesize;
bounce_buf = kmalloc(ops.len, GFP_KERNEL);
if (!bounce_buf) {
ret = -ENOMEM;
goto out;
}
for (;;) {
if (!is_buffer_in_page(buf, ops.len)) {
memcpy(bounce_buf, buf, ops.len);
ops.datbuf = (uint8_t *) bounce_buf;
} else {
ops.datbuf = (uint8_t *) buf;
}
ret = msm_nand_write_oob(mtd, to, &ops);
if (ret < 0)
break;
len -= mtd->writesize;
*retlen += mtd->writesize;
if (len == 0)
break;
buf += mtd->writesize;
to += mtd->writesize;
}
kfree(bounce_buf);
} else {
ops.len = len;
ops.datbuf = (uint8_t *)buf;
ret = msm_nand_write_oob(mtd, to, &ops);
*retlen = ops.retlen;
}
out:
return ret;
}
/*
* Structure that contains NANDc register data for commands required
* for Erase operation.
*/
struct msm_nand_erase_reg_data {
struct msm_nand_common_cfgs cfg;
uint32_t exec;
uint32_t flash_status;
uint32_t clrfstatus;
uint32_t clrrstatus;
};
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to erase a
* block within NAND device.
*/
#define ERASE_CMDS 9
static int msm_nand_erase(struct mtd_info *mtd, struct erase_info *instr)
{
int i = 0, err = 0;
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
uint32_t page = 0;
struct msm_nand_sps_cmd *cmd, *curr_cmd;
struct msm_nand_erase_reg_data data;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
ktime_t start;
/*
* The following 9 commands are required to erase a page -
* flash, addr0, addr1, cfg0, cfg1, exec, flash_status(read),
* flash_status(write), read_status.
*/
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[ERASE_CMDS];
struct msm_nand_sps_cmd cmd[ERASE_CMDS];
uint32_t flash_status;
} *dma_buffer;
if (unlikely(enable_perfstats))
start = ktime_get();
if (mtd->writesize == PAGE_SIZE_2K)
page = instr->addr >> 11;
if (mtd->writesize == PAGE_SIZE_4K)
page = instr->addr >> 12;
if (instr->addr & (mtd->erasesize - 1)) {
pr_err("unsupported erase address, 0x%llx\n", instr->addr);
err = -EINVAL;
goto out;
}
if (instr->len != mtd->erasesize) {
pr_err("unsupported erase len, %lld\n", instr->len);
err = -EINVAL;
goto out;
}
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer(
chip, sizeof(*dma_buffer))));
cmd = dma_buffer->cmd;
memset(&data, 0, sizeof(struct msm_nand_erase_reg_data));
data.cfg.cmd = MSM_NAND_CMD_BLOCK_ERASE;
data.cfg.addr0 = page;
data.cfg.addr1 = 0;
data.cfg.cfg0 = chip->cfg0 & (~(7 << CW_PER_PAGE));
data.cfg.cfg1 = chip->cfg1;
data.exec = 1;
dma_buffer->flash_status = 0xeeeeeeee;
data.clrfstatus = MSM_NAND_RESET_FLASH_STS;
data.clrrstatus = MSM_NAND_RESET_READ_STS;
curr_cmd = cmd;
msm_nand_prep_cfg_cmd_desc(info, data.cfg, &curr_cmd);
cmd = curr_cmd;
msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE,
data.exec, SPS_IOVEC_FLAG_NWD);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), READ,
msm_virt_to_dma(chip, &dma_buffer->flash_status), 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), WRITE,
data.clrfstatus, 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_READ_STATUS(info), WRITE,
data.clrrstatus,
SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_INT);
cmd++;
WARN_ON((cmd - dma_buffer->cmd) > ERASE_CMDS);
dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd);
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
for (i = 0; i < dma_buffer->xfer.iovec_count; i++) {
iovec->addr = msm_virt_to_dma(chip, &dma_buffer->cmd[i].ce);
iovec->size = sizeof(struct sps_command_element);
iovec->flags = dma_buffer->cmd[i].flags;
iovec++;
}
mutex_lock(&info->lock);
err = msm_nand_get_device(chip->dev);
if (err)
goto unlock_mutex;
err = sps_transfer(info->sps.cmd_pipe.handle, &dma_buffer->xfer);
if (err) {
pr_err("Failed to submit commands %d\n", err);
goto put_dev;
}
err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count,
&iovec_temp);
if (err) {
pr_err("Failed to get iovec for pipe %d (err: %d)\n",
(info->sps.cmd_pipe.index), err);
goto put_dev;
}
err = msm_nand_put_device(chip->dev);
if (err)
goto unlock_mutex;
/* Check for flash status errors */
if (dma_buffer->flash_status & (FS_OP_ERR |
FS_MPU_ERR | FS_DEVICE_STS_ERR)) {
pr_err("MPU/OP/DEV err (0x%x) set\n", dma_buffer->flash_status);
err = -EIO;
}
if (!(dma_buffer->flash_status & FS_DEVICE_WP)) {
pr_err("Device is write protected\n");
err = -EIO;
}
if (err) {
pr_err("Erase failed, 0x%llx\n", instr->addr);
instr->fail_addr = instr->addr;
instr->state = MTD_ERASE_FAILED;
} else {
instr->state = MTD_ERASE_DONE;
instr->fail_addr = 0xffffffff;
mtd_erase_callback(instr);
}
goto unlock_mutex;
put_dev:
msm_nand_put_device(chip->dev);
unlock_mutex:
mutex_unlock(&info->lock);
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer));
out:
if (unlikely(enable_perfstats) && likely(!err))
msm_nand_update_erase_perf_stats(info, start, 1);
return err;
}
/*
* Structure that contains NANDc register data for commands required
* for checking if a block is bad.
*/
struct msm_nand_blk_isbad_data {
struct msm_nand_common_cfgs cfg;
uint32_t ecc_bch_cfg;
uint32_t exec;
uint32_t read_offset;
};
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to check if
* a block is bad. This is done by reading the first page within a block and
* checking whether the bad block byte location contains 0xFF or not. If it
* doesn't contain 0xFF, then it is considered as bad block.
*/
#define ISBAD_CMDS 9
static int msm_nand_block_isbad(struct mtd_info *mtd, loff_t ofs)
{
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
int i = 0, ret = 0, bad_block = 0, submitted_num_desc = 1;
uint8_t *buf;
uint32_t page = 0, rdata, cwperpage;
struct msm_nand_sps_cmd *cmd, *curr_cmd;
struct msm_nand_blk_isbad_data data;
struct sps_iovec *iovec;
struct sps_iovec iovec_temp;
/*
* The following 9 commands are required to check bad block -
* flash, addr0, addr1, cfg0, cfg1, ecc_cfg, read_loc_0,
* exec, flash_status(read).
*/
struct {
struct sps_transfer xfer;
struct sps_iovec cmd_iovec[ISBAD_CMDS];
struct msm_nand_sps_cmd cmd[ISBAD_CMDS];
uint32_t flash_status;
} *dma_buffer;
if (mtd->writesize == PAGE_SIZE_2K)
page = ofs >> 11;
if (mtd->writesize == PAGE_SIZE_4K)
page = ofs >> 12;
cwperpage = (mtd->writesize >> 9);
if (ofs > mtd->size) {
pr_err("Invalid offset 0x%llx\n", ofs);
bad_block = -EINVAL;
goto out;
}
if (ofs & (mtd->erasesize - 1)) {
pr_err("unsupported block address, 0x%x\n", (uint32_t)ofs);
bad_block = -EINVAL;
goto out;
}
wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer(
chip, sizeof(*dma_buffer) + 4)));
buf = (uint8_t *)dma_buffer + sizeof(*dma_buffer);
cmd = dma_buffer->cmd;
memset(&data, 0, sizeof(struct msm_nand_blk_isbad_data));
data.cfg.cmd = MSM_NAND_CMD_PAGE_READ_ALL;
data.cfg.cfg0 = chip->cfg0_raw & ~(7U << CW_PER_PAGE);
data.cfg.cfg1 = chip->cfg1_raw;
if (chip->cfg1 & (1 << WIDE_FLASH))
data.cfg.addr0 = (page << 16) |
((chip->cw_size * (cwperpage-1)) >> 1);
else
data.cfg.addr0 = (page << 16) |
(chip->cw_size * (cwperpage-1));
data.cfg.addr1 = (page >> 16) & 0xff;
data.ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
data.exec = 1;
data.read_offset = (mtd->writesize - (chip->cw_size * (cwperpage-1)));
dma_buffer->flash_status = 0xeeeeeeee;
curr_cmd = cmd;
msm_nand_prep_cfg_cmd_desc(info, data.cfg, &curr_cmd);
cmd = curr_cmd;
msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_ECC_CFG(info), WRITE,
data.ecc_bch_cfg, 0);
cmd++;
rdata = (data.read_offset << 0) | (4 << 16) | (1 << 31);
msm_nand_prep_single_desc(cmd, MSM_NAND_READ_LOCATION_0(info), WRITE,
rdata, 0);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE,
data.exec, SPS_IOVEC_FLAG_NWD);
cmd++;
msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), READ,
msm_virt_to_dma(chip, &dma_buffer->flash_status),
SPS_IOVEC_FLAG_INT | SPS_IOVEC_FLAG_UNLOCK);
cmd++;
WARN_ON(cmd - dma_buffer->cmd > ISBAD_CMDS);
dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd);
dma_buffer->xfer.iovec = dma_buffer->cmd_iovec;
dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip,
&dma_buffer->cmd_iovec);
iovec = dma_buffer->xfer.iovec;
for (i = 0; i < dma_buffer->xfer.iovec_count; i++) {
iovec->addr = msm_virt_to_dma(chip, &dma_buffer->cmd[i].ce);
iovec->size = sizeof(struct sps_command_element);
iovec->flags = dma_buffer->cmd[i].flags;
iovec++;
}
mutex_lock(&info->lock);
ret = msm_nand_get_device(chip->dev);
if (ret) {
mutex_unlock(&info->lock);
goto free_dma;
}
/* Submit data descriptor */
ret = sps_transfer_one(info->sps.data_prod.handle,
msm_virt_to_dma(chip, buf),
4, NULL, SPS_IOVEC_FLAG_INT);
if (ret) {
pr_err("Failed to submit data desc %d\n", ret);
goto put_dev;
}
/* Submit command descriptor */
ret = sps_transfer(info->sps.cmd_pipe.handle, &dma_buffer->xfer);
if (ret) {
pr_err("Failed to submit commands %d\n", ret);
goto put_dev;
}
ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count,
&iovec_temp);
if (ret) {
pr_err("Failed to get iovec for pipe %d (ret: %d)\n",
(info->sps.cmd_pipe.index), ret);
goto put_dev;
}
ret = msm_nand_sps_get_iovec(info->sps.data_prod.handle,
info->sps.data_prod.index, submitted_num_desc,
&iovec_temp);
if (ret) {
pr_err("Failed to get iovec for pipe %d (ret: %d)\n",
(info->sps.data_prod.index), ret);
goto put_dev;
}
ret = msm_nand_put_device(chip->dev);
mutex_unlock(&info->lock);
if (ret)
goto free_dma;
/* Check for flash status errors */
if (dma_buffer->flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
pr_err("MPU/OP err set: %x\n", dma_buffer->flash_status);
bad_block = -EIO;
goto free_dma;
}
/* Check for bad block marker byte */
if (chip->cfg1 & (1 << WIDE_FLASH)) {
if (buf[0] != 0xFF || buf[1] != 0xFF)
bad_block = 1;
} else {
if (buf[0] != 0xFF)
bad_block = 1;
}
goto free_dma;
put_dev:
msm_nand_put_device(chip->dev);
mutex_unlock(&info->lock);
free_dma:
msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer) + 4);
out:
return ret ? ret : bad_block;
}
/*
* Function that gets called from upper layers such as MTD/YAFFS2 to mark a
* block as bad. This is done by writing the first page within a block with 0,
* thus setting the bad block byte location as well to 0.
*/
static int msm_nand_block_markbad(struct mtd_info *mtd, loff_t ofs)
{
struct mtd_oob_ops ops;
int ret;
uint8_t *buf;
size_t len;
if (ofs > mtd->size) {
pr_err("Invalid offset 0x%llx\n", ofs);
ret = -EINVAL;
goto out;
}
if (ofs & (mtd->erasesize - 1)) {
pr_err("unsupported block address, 0x%x\n", (uint32_t)ofs);
ret = -EINVAL;
goto out;
}
len = mtd->writesize + mtd->oobsize;
buf = kzalloc(len, GFP_KERNEL);
if (!buf) {
pr_err("unable to allocate memory for 0x%x size\n", len);
ret = -ENOMEM;
goto out;
}
ops.mode = MTD_OPS_RAW;
ops.len = len;
ops.retlen = 0;
ops.ooblen = 0;
ops.datbuf = buf;
ops.oobbuf = NULL;
ret = msm_nand_write_oob(mtd, ofs, &ops);
kfree(buf);
out:
return ret;
}
/*
* Function that scans for the attached NAND device. This fills out all
* the uninitialized function pointers with the defaults. The flash ID is
* read and the mtd/chip structures are filled with the appropriate values.
*/
static int msm_nand_scan(struct mtd_info *mtd)
{
struct msm_nand_info *info = mtd->priv;
struct msm_nand_chip *chip = &info->nand_chip;
struct flash_identification *supported_flash = &info->flash_dev;
int err = 0;
uint32_t i, j, mtd_writesize;
uint8_t dev_found = 0, wide_bus;
uint32_t manid, devid, devcfg;
uint32_t flash_id = 0, flash_id2 = 0;
uint8_t id_byte[NAND_MAX_ID_LEN];
uint32_t bad_block_byte, spare_bytes;
struct nand_flash_dev *flashdev = NULL;
struct nand_manufacturers *flashman = NULL;
/* Probe the Flash device for ONFI compliance */
if (!msm_nand_flash_onfi_probe(info)) {
dev_found = 1;
} else {
err = msm_nand_flash_read_id(info, 0, &flash_id, &flash_id2);
if (err < 0) {
pr_err("Failed to read Flash ID\n");
err = -EINVAL;
goto out;
}
manid = id_byte[0] = flash_id & 0xFF;
devid = id_byte[1] = (flash_id >> 8) & 0xFF;
devcfg = id_byte[3] = (flash_id >> 24) & 0xFF;
id_byte[2] = (flash_id >> 16) & 0xFF;
id_byte[4] = flash_id2 & 0xFF;
id_byte[5] = (flash_id2 >> 8) & 0xFF;
id_byte[6] = (flash_id2 >> 16) & 0xFF;
id_byte[7] = (flash_id2 >> 24) & 0xFF;
for (i = 0; !flashman && nand_manuf_ids[i].id; ++i)
if (nand_manuf_ids[i].id == manid)
flashman = &nand_manuf_ids[i];
for (i = 0; !flashdev && nand_flash_ids[i].id; ++i) {
/*
* If id_len is specified for an entry in the nand ids
* array, then at least 4 bytes of the nand id is
* present in the nand ids array - use that to identify
* the nand device first. If that is not present, only
* then fall back to searching the legacy or extended
* ids in the nand ids array.
* The id_len number of bytes in the nand id read from
* the device are checked against those in the nand id
* table for exact match.
*/
if (nand_flash_ids[i].id_len) {
for (j = 0; j < nand_flash_ids[i].id_len; j++) {
if (nand_flash_ids[i].id[j] ==
id_byte[j])
continue;
else
break;
}
if (j == nand_flash_ids[i].id_len)
flashdev = &nand_flash_ids[i];
} else if (!nand_flash_ids[i].id_len &&
nand_flash_ids[i].dev_id == devid)
flashdev = &nand_flash_ids[i];
}
if (!flashdev || !flashman) {
pr_err("unknown nand flashid=%x manuf=%x devid=%x\n",
flash_id, manid, devid);
err = -ENOENT;
goto out;
}
dev_found = 1;
if (!flashdev->pagesize) {
pr_err("missing page size info - extract from NAND ID\n");
supported_flash->widebus = devcfg & (1 << 6) ? 1 : 0;
supported_flash->pagesize = 1024 << (devcfg & 0x3);
supported_flash->blksize = (64 * 1024) <<
((devcfg >> 4) & 0x3);
supported_flash->oobsize = (8 << ((devcfg >> 2) & 1)) *
(supported_flash->pagesize >> 9);
} else {
supported_flash->widebus = flashdev->options &
NAND_BUSWIDTH_16 ? 1 : 0;
supported_flash->pagesize = flashdev->pagesize;
supported_flash->blksize = flashdev->erasesize;
supported_flash->oobsize = flashdev->oobsize;
supported_flash->ecc_correctability =
flashdev->ecc.strength_ds;
if (!flashdev->ecc.strength_ds)
pr_err("num ecc correctable bit not specified and defaults to 4 bit BCH\n");
}
supported_flash->flash_id = flash_id;
supported_flash->density = ((uint64_t)flashdev->chipsize) << 20;
}
if (dev_found) {
wide_bus = supported_flash->widebus;
mtd->size = supported_flash->density;
mtd->writesize = supported_flash->pagesize;
mtd->oobsize = supported_flash->oobsize;
mtd->erasesize = supported_flash->blksize;
mtd->writebufsize = mtd->writesize;
mtd_writesize = mtd->writesize;
/* Check whether NAND device support 8bit ECC*/
if (supported_flash->ecc_correctability >= 8) {
chip->bch_caps = MSM_NAND_CAP_8_BIT_BCH;
supported_flash->ecc_capability = 8;
} else {
chip->bch_caps = MSM_NAND_CAP_4_BIT_BCH;
supported_flash->ecc_capability = 4;
}
pr_info("NAND Id: 0x%x Buswidth: %dBits Density: %lld MByte\n",
supported_flash->flash_id, (wide_bus) ? 16 : 8,
(mtd->size >> 20));
pr_info("pagesize: %d Erasesize: %d oobsize: %d (in Bytes)\n",
mtd->writesize, mtd->erasesize, mtd->oobsize);
pr_info("BCH ECC: %d Bit\n", supported_flash->ecc_capability);
}
chip->cw_size = (chip->bch_caps & MSM_NAND_CAP_8_BIT_BCH) ? 532 : 528;
chip->cfg0 = (((mtd_writesize >> 9) - 1) << CW_PER_PAGE)
| (516 << UD_SIZE_BYTES)
| (0 << DISABLE_STATUS_AFTER_WRITE)
| (5 << NUM_ADDR_CYCLES);
bad_block_byte = (mtd_writesize - (chip->cw_size * (
(mtd_writesize >> 9) - 1)) + 1);
chip->cfg1 = (7 << NAND_RECOVERY_CYCLES)
| (0 << CS_ACTIVE_BSY)
| (bad_block_byte << BAD_BLOCK_BYTE_NUM)
| (0 << BAD_BLOCK_IN_SPARE_AREA)
| (2 << WR_RD_BSY_GAP)
| ((wide_bus ? 1 : 0) << WIDE_FLASH)
| (1 << ENABLE_BCH_ECC);
/*
* For 4bit BCH ECC (default ECC), parity bytes = 7(x8) or 8(x16 I/O)
* For 8bit BCH ECC, parity bytes = 13 (x8) or 14 (x16 I/O).
*/
chip->ecc_parity_bytes = (chip->bch_caps & MSM_NAND_CAP_8_BIT_BCH) ?
(wide_bus ? 14 : 13) : (wide_bus ? 8 : 7);
spare_bytes = chip->cw_size - (BYTES_512 + chip->ecc_parity_bytes);
chip->cfg0_raw = (((mtd_writesize >> 9) - 1) << CW_PER_PAGE)
| (5 << NUM_ADDR_CYCLES)
| (spare_bytes << SPARE_SIZE_BYTES)
| (BYTES_512 << UD_SIZE_BYTES);
chip->cfg1_raw = (2 << WR_RD_BSY_GAP)
| (1 << BAD_BLOCK_IN_SPARE_AREA)
| (21 << BAD_BLOCK_BYTE_NUM)
| (0 << CS_ACTIVE_BSY)
| (7 << NAND_RECOVERY_CYCLES)
| ((wide_bus ? 1 : 0) << WIDE_FLASH)
| (1 << DEV0_CFG1_ECC_DISABLE);
chip->ecc_bch_cfg = (0 << ECC_CFG_ECC_DISABLE)
| (0 << ECC_SW_RESET)
| (516 << ECC_NUM_DATA_BYTES)
| (chip->ecc_parity_bytes << ECC_PARITY_SIZE_BYTES)
| (1 << ECC_FORCE_CLK_OPEN);
chip->ecc_cfg_raw = (1 << ECC_FORCE_CLK_OPEN)
| (BYTES_512 << ECC_NUM_DATA_BYTES)
| (chip->ecc_parity_bytes << ECC_PARITY_SIZE_BYTES)
| (0 << ECC_SW_RESET)
| (1 << ECC_CFG_ECC_DISABLE);
if (chip->bch_caps & MSM_NAND_CAP_8_BIT_BCH) {
chip->cfg0 |= (wide_bus ? 0 << SPARE_SIZE_BYTES :
2 << SPARE_SIZE_BYTES);
chip->ecc_bch_cfg |= (1 << ECC_MODE);
chip->ecc_cfg_raw |= (1 << ECC_MODE);
} else {
chip->cfg0 |= (wide_bus ? 2 << SPARE_SIZE_BYTES :
4 << SPARE_SIZE_BYTES);
chip->ecc_bch_cfg |= (0 << ECC_MODE);
chip->ecc_cfg_raw |= (0 << ECC_MODE);
}
chip->ecc_buf_cfg = 0x203; /* No of bytes covered by ECC - 516 bytes */
pr_info("CFG0: 0x%08x, CFG1: 0x%08x\n"
" RAWCFG0: 0x%08x, RAWCFG1: 0x%08x\n"
" ECCBUFCFG: 0x%08x, ECCBCHCFG: 0x%08x\n"
" RAWECCCFG: 0x%08x, BAD BLOCK BYTE: 0x%08x\n",
chip->cfg0, chip->cfg1, chip->cfg0_raw, chip->cfg1_raw,
chip->ecc_buf_cfg, chip->ecc_bch_cfg,
chip->ecc_cfg_raw, bad_block_byte);
if (mtd->writesize == 2048)
mtd->oobavail = 16;
else if (mtd->writesize == 4096)
mtd->oobavail = 32;
else {
pr_err("Unsupported NAND pagesize: 0x%x\n", mtd->writesize);
err = -ENODEV;
goto out;
}
/* Fill in remaining MTD driver data */
mtd->type = MTD_NANDFLASH;
mtd->flags = MTD_CAP_NANDFLASH;
mtd->_erase = msm_nand_erase;
mtd->_block_isbad = msm_nand_block_isbad;
mtd->_block_markbad = msm_nand_block_markbad;
mtd->_read = msm_nand_read;
mtd->_write = msm_nand_write;
mtd->_read_oob = msm_nand_read_oob;
mtd->_write_oob = msm_nand_write_oob;
mtd->owner = THIS_MODULE;
out:
return err;
}
#define BAM_APPS_PIPE_LOCK_GRP0 0
#define BAM_APPS_PIPE_LOCK_GRP1 1
/*
* This function allocates, configures, connects an end point and
* also registers event notification for an end point. It also allocates
* DMA memory for descriptor FIFO of a pipe.
*/
static int msm_nand_init_endpoint(struct msm_nand_info *info,
struct msm_nand_sps_endpt *end_point,
uint32_t pipe_index)
{
int rc = 0;
struct sps_pipe *pipe_handle;
struct sps_connect *sps_config = &end_point->config;
struct sps_register_event *sps_event = &end_point->event;
pipe_handle = sps_alloc_endpoint();
if (!pipe_handle) {
pr_err("sps_alloc_endpoint() failed\n");
rc = -ENOMEM;
goto out;
}
rc = sps_get_config(pipe_handle, sps_config);
if (rc) {
pr_err("sps_get_config() failed %d\n", rc);
goto free_endpoint;
}
if (pipe_index == SPS_DATA_PROD_PIPE_INDEX) {
/* READ CASE: source - BAM; destination - system memory */
sps_config->source = info->sps.bam_handle;
sps_config->destination = SPS_DEV_HANDLE_MEM;
sps_config->mode = SPS_MODE_SRC;
sps_config->src_pipe_index = pipe_index;
} else if (pipe_index == SPS_DATA_CONS_PIPE_INDEX ||
pipe_index == SPS_CMD_CONS_PIPE_INDEX) {
/* WRITE CASE: source - system memory; destination - BAM */
sps_config->source = SPS_DEV_HANDLE_MEM;
sps_config->destination = info->sps.bam_handle;
sps_config->mode = SPS_MODE_DEST;
sps_config->dest_pipe_index = pipe_index;
}
sps_config->options = SPS_O_AUTO_ENABLE | SPS_O_POLL |
SPS_O_ACK_TRANSFERS;
if (pipe_index == SPS_DATA_PROD_PIPE_INDEX ||
pipe_index == SPS_DATA_CONS_PIPE_INDEX)
sps_config->lock_group = BAM_APPS_PIPE_LOCK_GRP0;
else if (pipe_index == SPS_CMD_CONS_PIPE_INDEX)
sps_config->lock_group = BAM_APPS_PIPE_LOCK_GRP1;
/*
* Descriptor FIFO is a cyclic FIFO. If SPS_MAX_DESC_NUM descriptors
* are allowed to be submitted before we get any ack for any of them,
* the descriptor FIFO size should be: (SPS_MAX_DESC_NUM + 1) *
* sizeof(struct sps_iovec).
*/
sps_config->desc.size = (SPS_MAX_DESC_NUM + 1) *
sizeof(struct sps_iovec);
sps_config->desc.base = dmam_alloc_coherent(info->nand_chip.dev,
sps_config->desc.size,
&sps_config->desc.phys_base,
GFP_KERNEL);
if (!sps_config->desc.base) {
pr_err("dmam_alloc_coherent() failed for size %x\n",
sps_config->desc.size);
rc = -ENOMEM;
goto free_endpoint;
}
memset(sps_config->desc.base, 0x00, sps_config->desc.size);
rc = sps_connect(pipe_handle, sps_config);
if (rc) {
pr_err("sps_connect() failed %d\n", rc);
goto free_endpoint;
}
sps_event->options = SPS_O_EOT;
sps_event->mode = SPS_TRIGGER_WAIT;
sps_event->user = (void *)info;
rc = sps_register_event(pipe_handle, sps_event);
if (rc) {
pr_err("sps_register_event() failed %d\n", rc);
goto sps_disconnect;
}
end_point->index = pipe_index;
end_point->handle = pipe_handle;
pr_debug("pipe handle 0x%x for pipe %d\n", (uint32_t)pipe_handle,
pipe_index);
goto out;
sps_disconnect:
sps_disconnect(pipe_handle);
free_endpoint:
sps_free_endpoint(pipe_handle);
out:
return rc;
}
/* This function disconnects and frees an end point */
static void msm_nand_deinit_endpoint(struct msm_nand_info *info,
struct msm_nand_sps_endpt *end_point)
{
sps_disconnect(end_point->handle);
sps_free_endpoint(end_point->handle);
}
/*
* This function registers BAM device and initializes its end points for
* the following pipes -
* system consumer pipe for data (pipe#0),
* system producer pipe for data (pipe#1),
* system consumer pipe for commands (pipe#2).
*/
static int msm_nand_bam_init(struct msm_nand_info *nand_info)
{
struct sps_bam_props bam = {0};
int rc = 0;
bam.phys_addr = nand_info->bam_phys;
bam.virt_addr = nand_info->bam_base;
bam.irq = nand_info->bam_irq;
/*
* NAND device is accessible from both Apps and Modem processor and
* thus, NANDc and BAM are shared between both the processors. But BAM
* must be enabled and instantiated only once during boot up by
* Trustzone before Modem/Apps is brought out from reset.
*
* This is indicated to SPS driver on Apps by marking flag
* SPS_BAM_MGR_DEVICE_REMOTE. The following are the global
* initializations that will be done by Trustzone - Execution
* Environment, Pipes assignment to Apps/Modem, Pipe Super groups and
* Descriptor summing threshold.
*
* NANDc BAM device supports 2 execution environments - Modem and Apps
* and thus the flag SPS_BAM_MGR_MULTI_EE is set.
*/
bam.manage = SPS_BAM_MGR_DEVICE_REMOTE | SPS_BAM_MGR_MULTI_EE;
bam.ipc_loglevel = QPIC_BAM_DEFAULT_IPC_LOGLVL;
rc = sps_phy2h(bam.phys_addr, &nand_info->sps.bam_handle);
if (!rc)
goto init_sps_ep;
rc = sps_register_bam_device(&bam, &nand_info->sps.bam_handle);
if (rc) {
pr_err("%s: sps_register_bam_device() failed with %d\n",
__func__, rc);
goto out;
}
pr_info("%s: BAM device registered: bam_handle 0x%lx\n",
__func__, nand_info->sps.bam_handle);
init_sps_ep:
rc = msm_nand_init_endpoint(nand_info, &nand_info->sps.data_prod,
SPS_DATA_PROD_PIPE_INDEX);
if (rc)
goto out;
rc = msm_nand_init_endpoint(nand_info, &nand_info->sps.data_cons,
SPS_DATA_CONS_PIPE_INDEX);
if (rc)
goto deinit_data_prod;
rc = msm_nand_init_endpoint(nand_info, &nand_info->sps.cmd_pipe,
SPS_CMD_CONS_PIPE_INDEX);
if (rc)
goto deinit_data_cons;
goto out;
deinit_data_cons:
msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_cons);
deinit_data_prod:
msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_prod);
out:
return rc;
}
/*
* This function disconnects and frees its end points for all the pipes.
* Since the BAM is shared resource, it is not deregistered as its handle
* might be in use with LCDC.
*/
static void msm_nand_bam_free(struct msm_nand_info *nand_info)
{
msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_prod);
msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_cons);
msm_nand_deinit_endpoint(nand_info, &nand_info->sps.cmd_pipe);
}
/* This function enables DMA support for the NANDc in BAM mode. */
static int msm_nand_enable_dma(struct msm_nand_info *info)
{
struct msm_nand_sps_cmd *sps_cmd;
struct msm_nand_chip *chip = &info->nand_chip;
int ret, submitted_num_desc = 1;
struct sps_iovec iovec_temp;
wait_event(chip->dma_wait_queue,
(sps_cmd = msm_nand_get_dma_buffer(chip, sizeof(*sps_cmd))));
msm_nand_prep_single_desc(sps_cmd, MSM_NAND_CTRL(info), WRITE,
(1 << BAM_MODE_EN), SPS_IOVEC_FLAG_INT);
mutex_lock(&info->lock);
ret = msm_nand_get_device(chip->dev);
if (ret) {
mutex_unlock(&info->lock);
goto out;
}
ret = sps_transfer_one(info->sps.cmd_pipe.handle,
msm_virt_to_dma(chip, &sps_cmd->ce),
sizeof(struct sps_command_element), NULL,
sps_cmd->flags);
if (ret) {
pr_err("Failed to submit command: %d\n", ret);
goto put_dev;
}
ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle,
info->sps.cmd_pipe.index, submitted_num_desc,
&iovec_temp);
if (ret) {
pr_err("Failed to get iovec for pipe %d (ret: %d)\n",
(info->sps.cmd_pipe.index), ret);
goto put_dev;
}
put_dev:
ret = msm_nand_put_device(chip->dev);
out:
mutex_unlock(&info->lock);
msm_nand_release_dma_buffer(chip, sps_cmd, sizeof(*sps_cmd));
return ret;
}
static int msm_nand_parse_smem_ptable(int *nr_parts)
{
uint32_t i, j;
uint32_t len = FLASH_PTABLE_HDR_LEN;
struct flash_partition_entry *pentry;
char *delimiter = ":";
void *temp_ptable = NULL;
pr_info("Parsing partition table info from SMEM\n");
temp_ptable = smem_get_entry(SMEM_AARM_PARTITION_TABLE, &len, 0,
SMEM_ANY_HOST_FLAG);
if (IS_ERR_OR_NULL(temp_ptable)) {
pr_err("Error reading partition table header\n");
goto out;
}
/* Read only the header portion of ptable */
ptable = *(struct flash_partition_table *)temp_ptable;
/* Verify ptable magic */
if (ptable.magic1 != FLASH_PART_MAGIC1 ||
ptable.magic2 != FLASH_PART_MAGIC2) {
pr_err("Partition table magic verification failed\n");
goto out;
}
/* Ensure that # of partitions is less than the max we have allocated */
if (ptable.numparts > FLASH_PTABLE_MAX_PARTS_V4) {
pr_err("Partition numbers exceed the max limit\n");
goto out;
}
/* Find out length of partition data based on table version. */
if (ptable.version <= FLASH_PTABLE_V3) {
len = FLASH_PTABLE_HDR_LEN + FLASH_PTABLE_MAX_PARTS_V3 *
sizeof(struct flash_partition_entry);
} else if (ptable.version == FLASH_PTABLE_V4) {
len = FLASH_PTABLE_HDR_LEN + FLASH_PTABLE_MAX_PARTS_V4 *
sizeof(struct flash_partition_entry);
} else {
pr_err("Unknown ptable version (%d)", ptable.version);
goto out;
}
*nr_parts = ptable.numparts;
/*
* Now that the partition table header has been parsed, verified
* and the length of the partition table calculated, read the
* complete partition table.
*/
temp_ptable = smem_get_entry(SMEM_AARM_PARTITION_TABLE, &len, 0,
SMEM_ANY_HOST_FLAG);
if (IS_ERR_OR_NULL(temp_ptable)) {
pr_err("Error reading partition table\n");
goto out;
}
/* Read only the header portion of ptable */
ptable = *(struct flash_partition_table *)temp_ptable;
for (i = 0; i < ptable.numparts; i++) {
pentry = &ptable.part_entry[i];
if (pentry->name[0] == '\0')
continue;
/* Convert name to lower case and discard the initial chars */
mtd_part[i].name = pentry->name;
strsep(&(mtd_part[i].name), delimiter);
if (!mtd_part[i].name)
mtd_part[i].name = pentry->name;
for (j = 0; j < strlen(mtd_part[i].name); j++)
*(mtd_part[i].name + j) =
tolower(*(mtd_part[i].name + j));
mtd_part[i].offset = pentry->offset;
mtd_part[i].mask_flags = pentry->attr;
mtd_part[i].size = pentry->length;
pr_debug("%d: %s offs=0x%08x size=0x%08x attr:0x%08x\n",
i, pentry->name, pentry->offset, pentry->length,
pentry->attr);
}
pr_info("SMEM partition table found: ver: %d len: %d\n",
ptable.version, ptable.numparts);
return 0;
out:
return -EINVAL;
}
#define BOOT_DEV_MASK 0x1E
#define BOOT_DEV_NAND 0x4
/*
* This function gets called when its device named msm-nand is added to
* device tree .dts file with all its resources such as physical addresses
* for NANDc and BAM, BAM IRQ.
*
* It also expects the NAND flash partition information to be passed in .dts
* file so that it can parse the partitions by calling MTD function
* mtd_device_parse_register().
*
*/
static int msm_nand_probe(struct platform_device *pdev)
{
struct msm_nand_info *info;
struct resource *res;
int i, err, nr_parts;
struct device *dev;
u32 adjustment_offset;
void __iomem *boot_cfg_base;
u32 boot_dev;
res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"boot_cfg");
if (res && res->start) {
boot_cfg_base = devm_ioremap(&pdev->dev, res->start,
resource_size(res));
if (!boot_cfg_base) {
pr_err("ioremap() failed for addr 0x%x size 0x%x\n",
res->start, resource_size(res));
return -ENOMEM;
}
boot_dev = (readl_relaxed(boot_cfg_base) & BOOT_DEV_MASK) >> 1;
if (boot_dev != BOOT_DEV_NAND) {
pr_err("disabling nand as boot device (%x) is not NAND\n",
boot_dev);
return -ENODEV;
}
}
/*
* The partition information can also be passed from kernel command
* line. Also, the MTD core layer supports adding the whole device as
* one MTD device when no partition information is available at all.
*/
info = devm_kzalloc(&pdev->dev, sizeof(struct msm_nand_info),
GFP_KERNEL);
if (!info) {
err = -ENOMEM;
goto out;
}
res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"nand_phys");
if (!res || !res->start) {
pr_err("NAND phys address range is not provided\n");
err = -ENODEV;
goto out;
}
info->nand_phys = res->start;
err = of_property_read_u32(pdev->dev.of_node,
"qcom,reg-adjustment-offset",
&adjustment_offset);
if (err) {
pr_err("adjustment_offset not found, err = %d\n", err);
WARN_ON(1);
return err;
}
info->nand_phys_adjusted = info->nand_phys + adjustment_offset;
res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"bam_phys");
if (!res || !res->start) {
pr_err("BAM phys address range is not provided\n");
err = -ENODEV;
goto out;
}
info->bam_phys = res->start;
info->bam_base = devm_ioremap(&pdev->dev, res->start,
resource_size(res));
if (!info->bam_base) {
pr_err("BAM ioremap() failed for addr 0x%x size 0x%x\n",
res->start, resource_size(res));
err = -ENOMEM;
goto out;
}
info->bam_irq = platform_get_irq_byname(pdev, "bam_irq");
if (info->bam_irq < 0) {
pr_err("BAM IRQ is not provided\n");
err = -ENODEV;
goto out;
}
info->mtd.name = dev_name(&pdev->dev);
info->mtd.priv = info;
info->mtd.owner = THIS_MODULE;
info->nand_chip.dev = &pdev->dev;
init_waitqueue_head(&info->nand_chip.dma_wait_queue);
mutex_init(&info->lock);
dev = &pdev->dev;
if (dma_supported(dev, DMA_BIT_MASK(32))) {
info->dma_mask = DMA_BIT_MASK(32);
dev->coherent_dma_mask = info->dma_mask;
}
info->nand_chip.dma_virt_addr =
dmam_alloc_coherent(&pdev->dev, MSM_NAND_DMA_BUFFER_SIZE,
&info->nand_chip.dma_phys_addr, GFP_KERNEL);
if (!info->nand_chip.dma_virt_addr) {
pr_err("No memory for DMA buffer size %x\n",
MSM_NAND_DMA_BUFFER_SIZE);
err = -ENOMEM;
goto out;
}
err = msm_nand_bus_register(pdev, info);
if (err)
goto out;
if (of_property_read_bool(pdev->dev.of_node, "qcom,qpic-clk-rpmh"))
info->clk_data.rpmh_clk = true;
if (!info->clk_data.rpmh_clk) {
info->clk_data.qpic_clk = devm_clk_get(&pdev->dev, "core_clk");
if (!IS_ERR_OR_NULL(info->clk_data.qpic_clk)) {
err = clk_set_rate(info->clk_data.qpic_clk,
MSM_NAND_BUS_VOTE_MAX_RATE);
} else {
err = PTR_ERR(info->clk_data.qpic_clk);
pr_err("Failed to get clock handle, err=%d\n", err);
}
if (err)
goto bus_unregister;
}
err = msm_nand_setup_clocks_and_bus_bw(info, true);
if (err)
goto bus_unregister;
dev_set_drvdata(&pdev->dev, info);
err = pm_runtime_set_active(&pdev->dev);
if (err)
pr_err("pm_runtime_set_active() failed with error %d", err);
pm_runtime_enable(&pdev->dev);
pm_runtime_use_autosuspend(&pdev->dev);
pm_runtime_set_autosuspend_delay(&pdev->dev, MSM_NAND_IDLE_TIMEOUT);
err = msm_nand_bam_init(info);
if (err) {
pr_err("msm_nand_bam_init() failed %d\n", err);
goto clk_rpm_disable;
}
err = msm_nand_enable_dma(info);
if (err) {
pr_err("Failed to enable DMA in NANDc\n");
goto free_bam;
}
err = msm_nand_parse_smem_ptable(&nr_parts);
if (err < 0) {
pr_err("Failed to parse partition table in SMEM\n");
goto free_bam;
}
if (msm_nand_scan(&info->mtd)) {
pr_err("No nand device found\n");
err = -ENXIO;
goto free_bam;
}
for (i = 0; i < nr_parts; i++) {
mtd_part[i].offset *= info->mtd.erasesize;
mtd_part[i].size *= info->mtd.erasesize;
}
err = mtd_device_parse_register(&info->mtd, NULL, NULL,
&mtd_part[0], nr_parts);
if (err < 0) {
pr_err("Unable to register MTD partitions %d\n", err);
goto free_bam;
}
pr_info("NANDc phys addr 0x%lx, BAM phys addr 0x%lx, BAM IRQ %d\n",
info->nand_phys, info->bam_phys, info->bam_irq);
pr_info("Allocated DMA buffer at virt_addr 0x%pK, phys_addr 0x%x\n",
info->nand_chip.dma_virt_addr, info->nand_chip.dma_phys_addr);
msm_nand_init_sysfs(dev);
msm_nand_init_perf_stats(info);
goto out;
free_bam:
msm_nand_bam_free(info);
clk_rpm_disable:
msm_nand_setup_clocks_and_bus_bw(info, false);
pm_runtime_disable(&(pdev)->dev);
pm_runtime_set_suspended(&(pdev)->dev);
bus_unregister:
msm_nand_bus_unregister(info);
out:
return err;
}
/*
* Remove functionality that gets called when driver/device msm-nand
* is removed.
*/
static int msm_nand_remove(struct platform_device *pdev)
{
struct msm_nand_info *info = dev_get_drvdata(&pdev->dev);
msm_nand_cleanup_sysfs(&pdev->dev);
if (pm_runtime_suspended(&(pdev)->dev))
pm_runtime_resume(&(pdev)->dev);
pm_runtime_disable(&(pdev)->dev);
pm_runtime_set_suspended(&(pdev)->dev);
dev_set_drvdata(&pdev->dev, NULL);
if (info) {
msm_nand_setup_clocks_and_bus_bw(info, false);
if (info->clk_data.client_handle)
msm_nand_bus_unregister(info);
mtd_device_unregister(&info->mtd);
msm_nand_bam_free(info);
}
return 0;
}
#define DRIVER_NAME "msm_qpic_nand"
static const struct of_device_id msm_nand_match_table[] = {
{ .compatible = "qcom,msm-nand", },
{},
};
static const struct dev_pm_ops msm_nand_pm_ops = {
.suspend = msm_nand_suspend,
.resume = msm_nand_resume,
.runtime_suspend = msm_nand_runtime_suspend,
.runtime_resume = msm_nand_runtime_resume,
};
static struct platform_driver msm_nand_driver = {
.probe = msm_nand_probe,
.remove = msm_nand_remove,
.driver = {
.name = DRIVER_NAME,
.of_match_table = msm_nand_match_table,
.pm = &msm_nand_pm_ops,
},
};
module_param(enable_euclean, bool, 0644);
MODULE_PARM_DESC(enable_euclean, "Set this parameter to enable reporting EUCLEAN to upper layer when the correctable bitflips are equal to the max correctable limit.");
module_param(enable_perfstats, bool, 0644);
MODULE_PARM_DESC(enable_perfstats, "Set this parameter to enable collection and reporting of performance data.");
module_platform_driver(msm_nand_driver);
MODULE_ALIAS(DRIVER_NAME);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("MSM QPIC NAND flash driver");