diff --git a/sound/soc/codecs/twl6040.c b/sound/soc/codecs/twl6040.c
index 284dd2e..836eb14 100644
--- a/sound/soc/codecs/twl6040.c
+++ b/sound/soc/codecs/twl6040.c
@@ -1540,7 +1540,6 @@
 
 	priv->codec = codec;
 	codec->control_data = dev_get_drvdata(codec->dev->parent);
-	codec->ignore_pmdown_time = 1;
 
 	if (pdata && pdata->hs_left_step && pdata->hs_right_step) {
 		priv->hs_left_step = pdata->hs_left_step;
@@ -1626,6 +1625,7 @@
 	.reg_cache_size = ARRAY_SIZE(twl6040_reg),
 	.reg_word_size = sizeof(u8),
 	.reg_cache_default = twl6040_reg,
+	.ignore_pmdown_time = true,
 
 	.controls = twl6040_snd_controls,
 	.num_controls = ARRAY_SIZE(twl6040_snd_controls),