import numpy as np
import pandas as pd

np.random.seed(123)

df = pd.DataFrame({"M2018" : {0 : "p", 1 : "q", 2 : "r"},
                   "M2018" : {0 : "s", 1 : "t", 2 : "u"},
                   "N2019" : {0 : 2.5, 1 : 1.2, 2 : .7},
                   "N2019" : {0 : 3.2, 1 : 1.3, 2 : .1},
                   "X"     : dict(zip(range(3), np.random.randn(3)))
                   })

df["id"] = df.index
df

	M2018	N2019	X	id
0	s	3.2	-1.085631	0
1	t	1.3	0.997345	1
2	u	0.1	0.282978	2

pd.wide_to_long(df, ["M", "N"], i="id", j="year")
... # doctest: +NORMALIZE_WHITESPACE

		X	M	N
id	year
0	2018	-1.085631	s	NaN
	2019	-1.085631	NaN	3.2
1	2018	0.997345	t	NaN
	2019	0.997345	NaN	1.3
2	2018	0.282978	u	NaN
	2019	0.282978	NaN	0.1

With multiple id columns

df = pd.DataFrame({
    'fID': [1, 1, 2, 2, 3, 3],
    'birth': [1, 2, 3, 1, 2, 3],
    'ht1': [2.8, 2.9, 2.2, 2, 1.8, 1.9],
    'ht2': [3.4, 3.8, 2.9, 3.2, 2.8, 2.4]
 })
df

	fID	birth	ht1	ht2
0	1	1	2.8	3.4
1	1	2	2.9	3.8
2	2	3	2.2	2.9
3	2	1	2.0	3.2
4	3	2	1.8	2.8
5	3	3	1.9	2.4

y = pd.wide_to_long(df, stubnames='ht', i=['fID', 'birth'], j='age')
y
# doctest: +NORMALIZE_WHITESPACE

			ht
fID	birth	age
1	1	1	2.8
		2	3.4
	2	1	2.9
		2	3.8
2	3	1	2.2
		2	2.9
	1	1	2.0
		2	3.2
3	2	1	1.8
		2	2.8
	3	1	1.9
		2	2.4

Going from long back to wide just takes some creative use of unstack

w = y.unstack()

w.columns = w.columns.map('{0[0]}{0[1]}'.format)

w.reset_index()

	fID	birth	ht1	ht2
0	1	1	2.8	3.4
1	1	2	2.9	3.8
2	2	1	2.0	3.2
3	2	3	2.2	2.9
4	3	2	1.8	2.8
5	3	3	1.9	2.4

Less wieldy column names are also handled:

np.random.seed(0)

df = pd.DataFrame({'P(weekly)-2015': np.random.rand(3),
                   'P(weekly)-2016': np.random.rand(3),
                   'Q(weekly)-2015': np.random.rand(3),
                   'Q(weekly)-2016': np.random.rand(3),
                   'X' : np.random.randint(3, size=3)})

df['id'] = df.index

df # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS

	P(weekly)-2015	P(weekly)-2016	Q(weekly)-2015	Q(weekly)-2016	X	id
0	0.548814	0.544883	0.437587	0.383442	0	0
1	0.715189	0.423655	0.891773	0.791725	1	1
2	0.602763	0.645894	0.963663	0.528895	1	2

pd.wide_to_long(df, ['P(weekly)', 'Q(weekly)'], i='id',
                j='year', sep='-')
# doctest: +NORMALIZE_WHITESPACE

		X	P(weekly)	Q(weekly)
id	year
0	2015	0	0.548814	0.437587
1	2015	1	0.715189	0.891773
2	2015	1	0.602763	0.963663
0	2016	0	0.544883	0.383442
1	2016	1	0.423655	0.791725
2	2016	1	0.645894	0.528895

If there are many columns you can use a regex to find our stubnames and pass that list on to
wide_to_long

stubnames = sorted(
     set([match[0] for match in df.columns.str.findall(
         r'[P-Q]\(.*\)').values if match != [] ])
 )

list(stubnames)

['P(weekly)', 'Q(weekly)']

All of the above examples have integers as suffixes. It is possible to have non-integers as suffixes.

df = pd.DataFrame({
    'fID': [1, 1, 2, 2, 3, 3],
    'birth': [1, 2, 3, 1, 2, 3],
    'ht_one': [2.8, 2.9, 2.2, 2, 1.8, 1.9],
    'ht_two': [3.4, 3.8, 2.9, 3.2, 2.8, 2.4]
 })
df

	fID	birth	ht_one	ht_two
0	1	1	2.8	3.4
1	1	2	2.9	3.8
2	2	3	2.2	2.9
3	2	1	2.0	3.2
4	3	2	1.8	2.8
5	3	3	1.9	2.4

y = pd.wide_to_long(df, stubnames='ht', i=['fID', 'birth'], j='age',
                    sep='_', suffix='\w+')
y
# doctest: +NORMALIZE_WHITESPACE

			ht
fID	birth	age
1	1	one	2.8
		two	3.4
	2	one	2.9
		two	3.8
2	3	one	2.2
		two	2.9
	1	one	2.0
		two	3.2
3	2	one	1.8
		two	2.8
	3	one	1.9
		two	2.4