【ADHD血液診断】データのダウンロードと前処理

import time

import pickle

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

from scipy.stats import pearsonr

from sklearn.decomposition import PCA

from sklearn.manifold import TSNE

import umap

from scipy.sparse.csgraph import connected_components

import warnings

warnings.filterwarnings('ignore')

df = pd.read_csv("src/GSE159104_DGE_raw_154samples.txt", sep="\t")

df

df.shape

(195178, 162)

df_selected_columns = df.iloc[:,1].tolist()

df_selected = df.iloc[:, 8:].T

df_selected.columns = df_selected_columns

df_selected.shape

(154, 195178)

df_selected

data = []

for i in range(df_selected.shape[0]):

    v = df_selected.iloc[i, :].tolist()

    s = np.sum(v)

    v_ = [(j/s)*(10**6) for j in v]

    data.append(v_)

df_rpkm = pd.DataFrame(data)

df_rpkm.index = df_selected.index

df_rpkm.columns = df_selected.columns

df_rpkm.shape

(154, 195178)

idxes = []

for i in range(df_rpkm.shape[1]):

    if not np.sum(df_rpkm.iloc[:,i]) <= 100:

        idxes.append(i)

df_excluded = df_rpkm.iloc[:, idxes]

df_excluded.shape

(154, 92807)

gene_set = list(set(df_excluded.columns.tolist()))

data = []

for i, g in enumerate(gene_set):

    tab = df_excluded.loc[:, g]

    if len(tab.shape) == 1:

        v = tab.tolist()

    else:

        v = df_excluded.loc[:, g].apply(lambda x: np.mean(x), axis=1).tolist()

    data.append(v)

df_gene = pd.DataFrame(data).T

df_gene.index = df_excluded.index

df_gene.columns = gene_set

df_gene.shape

(154, 16177)

non_zero_list = []

for i in range(df_gene.shape[1]):

    v = df_selected.iloc[:, i].tolist()

    s = np.std(v)

    if s != 0:

        non_zero_list.append(i)

df_std = df_gene.iloc[:, non_zero_list]

df_std.to_csv("src/df_std.csv")

df_std.shape

(154, 16042)