1.3_concat_different_samples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import anndata as ad\n",
    "import scanpy as sc\n",
    "import numpy as np\n",
    "import scipy as scp\n",
    "import pandas as pd\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[AnnData object with n_obs × n_vars = 23216 × 60623 \n",
       "     layers: 'spliced', 'unspliced',\n",
       " AnnData object with n_obs × n_vars = 22549 × 60623 \n",
       "     layers: 'spliced', 'unspliced',\n",
       " AnnData object with n_obs × n_vars = 22840 × 60623 \n",
       "     layers: 'spliced', 'unspliced',\n",
       " AnnData object with n_obs × n_vars = 22751 × 60623 \n",
       "     layers: 'spliced', 'unspliced',\n",
       " AnnData object with n_obs × n_vars = 23142 × 60623 \n",
       "     layers: 'spliced', 'unspliced',\n",
       " AnnData object with n_obs × n_vars = 23215 × 60623 \n",
       "     layers: 'spliced', 'unspliced']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adataNames = os.listdir('./jointAdata/')\n",
    "adataList = []\n",
    "for aN in adataNames[6:]:\n",
    "    adataList.append(sc.read_h5ad('./jointAdata/'+aN))\n",
    "adataList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['S4', 'S17', 'S16', 'S5', 'S15', 'S2', 'S13', 'S1', 'S3', 'S18', 'S6', 'S14']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples = []\n",
    "for an in adataNames:\n",
    "    samples.append(re.search('S[0-9]+.',an).group()[:-1])\n",
    "samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23914, 4)\n",
      "(23573, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "23415"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove duplicated values that have different conditions\n",
    "phenoDat = pd.read_csv('phenoSCCall.csv')\n",
    "phenoDat.drop('Unnamed: 0',axis=1, inplace=True)\n",
    "print(phenoDat.shape)\n",
    "dup = phenoDat.duplicated('barcodes',keep=False)\n",
    "dupPhenoDat = phenoDat.loc[dup,:]\n",
    "dup2 = dupPhenoDat.duplicated(['barcodes','treatment'],keep=False)\n",
    "dupKeep = dupPhenoDat.loc[dup2,:].sort_values('barcodes')\n",
    "remove = [x for x in list(dupPhenoDat.index) if x not in list(dupKeep.index)]\n",
    "pD = phenoDat.drop(remove,axis = 0)\n",
    "print(pD.shape)\n",
    "len(set(pD['barcodes']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AAACCTGAGTTGTCGT',\n",
       " 'AAACCTGCACAGCGTC',\n",
       " 'AAACCTGGTACTTCTT',\n",
       " 'AAAGATGGTGTGAATA',\n",
       " 'AAAGCAAGTAGCTGCC']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keep = list(pD['barcodes'])\n",
    "keep[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, range(2, 6)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in [0,range(2,6)]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "converting to array\n",
      "adding values\n",
      "done 0 of file no 1 of 2\n",
      "done 10000 of file no 1 of 2\n",
      "done 20000 of file no 1 of 2\n",
      "done 0 of file no 2 of 2\n",
      "done 10000 of file no 2 of 2\n",
      "done 20000 of file no 2 of 2\n",
      "converting into sparse matrices\n",
      "final adata is:\n",
      "AnnData object with n_obs × n_vars = 23585 × 60623 \n",
      "    layers: 'spliced', 'unspliced'\n",
      "writing file finalConcatanatedData.h5ad\n",
      "CPU times: user 1min 6s, sys: 45.5 s, total: 1min 52s\n",
      "Wall time: 1min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# here use all, subset with keep later\n",
    "checkSameVar = []\n",
    "for i in range(1,len(adataList)):\n",
    "    checkSameVar.append(all(adataList[i].var.index == adataList[i-1].var.index))\n",
    "check = all(checkSameVar)\n",
    "assert check == True, \"The vars are not same in all adata\"\n",
    "#convert each sparse matrix to array\n",
    "print('converting to array')\n",
    "XList = []\n",
    "SList = []\n",
    "UList = []\n",
    "obs = []\n",
    "var_names = list(adataList[0].var.index)\n",
    "obsNames = []\n",
    "for an in adataList:\n",
    "    XList.append(an.X.toarray())\n",
    "    SList.append(an.layers['spliced'].toarray())\n",
    "    UList.append(an.layers['unspliced'].toarray())\n",
    "    obsNames += list(an.obs.index.values)\n",
    "cellBarcodes = set(obsNames)\n",
    "rows = len(cellBarcodes)\n",
    "cols = adataList[0].shape[1]\n",
    "cellDict = dict(zip(cellBarcodes,range(rows)))\n",
    "X = np.zeros((rows,cols))\n",
    "S = np.zeros((rows,cols))\n",
    "U = np.zeros((rows,cols))\n",
    "print('adding values')\n",
    "for i in range(len(adataList)):\n",
    "    for j in range(adataList[i].shape[0]):\n",
    "        if j % 10000 == 0:\n",
    "            print('done ' + str(j) + ' of file no ' + str(i+1) + ' of ' + str(len(adataList)))\n",
    "        idx = cellDict[adataList[i].obs.index[j]]\n",
    "        X[idx] = X[idx] + XList[i][j]\n",
    "        S[idx] = S[idx] + SList[i][j]\n",
    "        U[idx] = U[idx] + UList[i][j]\n",
    "print('converting into sparse matrices')\n",
    "X = scp.sparse.csr_matrix(X)\n",
    "S = scp.sparse.csr_matrix(S)\n",
    "U = scp.sparse.csr_matrix(U)\n",
    "concatAdata = ad.AnnData(X,\n",
    "              {'obs_names': cellBarcodes},\n",
    "              {'var_names': var_names},\n",
    "               layers={'spliced':S,\n",
    "                      'unspliced':U})\n",
    "filename = 'finalConcatanatedData.h5ad'\n",
    "print('final adata is:')\n",
    "print(concatAdata)\n",
    "print('writing file ' + filename)\n",
    "concatAdata.write(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "adataList = [ad.read_h5ad('finalConcatanatedData_1.h5ad'),ad.read_h5ad('finalConcatanatedData_2.h5ad')]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "(Fertig) Python 3.7",
   "language": "python",
   "name": "fertig_python_3_7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}