diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 3460617..d69d098 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -302,14 +302,6 @@
 	if (has_ucv)
 		dw8250_setup_port(up);
 
-	/* if we have a valid fifosize, try hooking up DMA here */
-	if (p->fifosize) {
-		up->dma = &data->dma;
-
-		up->dma->rxconf.src_maxburst = p->fifosize / 4;
-		up->dma->txconf.dst_maxburst = p->fifosize / 4;
-	}
-
 	/* get index of serial line, if found in DT aliases */
 	id = of_alias_get_id(np, "serial");
 	if (id >= 0)
@@ -348,10 +340,6 @@
 		data->dma.fn = dw8250_idma_filter;
 	}
 
-	up->dma = &data->dma;
-	up->dma->rxconf.src_maxburst = p->fifosize / 4;
-	up->dma->txconf.dst_maxburst = p->fifosize / 4;
-
 	up->port.set_termios = dw8250_set_termios;
 
 	return 0;
@@ -498,6 +486,13 @@
 		goto err_reset;
 	}
 
+	/* If we have a valid fifosize, try hooking up DMA */
+	if (p->fifosize) {
+		data->dma.rxconf.src_maxburst = p->fifosize / 4;
+		data->dma.txconf.dst_maxburst = p->fifosize / 4;
+		uart.dma = &data->dma;
+	}
+
 	data->line = serial8250_register_8250_port(&uart);
 	if (data->line < 0) {
 		err = data->line;