Integrate scRNA-seq datasets#

Hide code cell content 
!lamin load test-scrna

💡 found cached instance metadata: /home/runner/.lamin/instance--testuser1--test-scrna.env

✅ loaded instance: testuser1/test-scrna


import lamindb as ln
import lnschema_bionty as lb
import pandas as pd
import anndata as ad

✅ loaded instance: testuser1/test-scrna (lamindb 0.50.7)

ln.track()

💡 notebook imports: anndata==0.9.2 lamindb==0.50.7 lnschema_bionty==0.29.6 pandas==1.5.3

✅ saved: Transform(id='agayZTonayqAz8', name='Integrate scRNA-seq datasets', short_name='scrna2', stem_id='agayZTonayqA', version='0', type=notebook, updated_at=2023-08-17 17:33:29, created_by_id='DzTjkKse')

✅ saved: Run(id='ZdrKUiFPGdQnnd9FGsEa', run_at=2023-08-17 17:33:29, transform_id='agayZTonayqAz8', created_by_id='DzTjkKse')

Query files based on metadata#

ln.File.filter(tissues__name__icontains="lymph node").distinct().df()
storage_id key suffix accessor description version initial_version_id size hash hash_type transform_id run_id updated_at created_by_id
id
9SaWYjVX2DwR0dCC2l1B 6ObKC1Wo None .h5ad AnnData Detmar22 None None 17342743 lC0PrTQici8k6yWBFsaJzg md5 Nv48yAceNSh8z8 UcskU0EgGleETR6Oao82 2023-08-17 17:32:54 DzTjkKse
P8fI3RY9UtTAEQfY7wx1 6ObKC1Wo None .h5ad AnnData Conde22 None None 28061905 3cIcmoqp1MxjX8NlRkKGlQ md5 Nv48yAceNSh8z8 UcskU0EgGleETR6Oao82 2023-08-17 17:33:12 DzTjkKse 
ln.File.filter(cell_types__name__icontains="monocyte").distinct().df()
storage_id key suffix accessor description version initial_version_id size hash hash_type transform_id run_id updated_at created_by_id
id
P8fI3RY9UtTAEQfY7wx1 6ObKC1Wo None .h5ad AnnData Conde22 None None 28061905 3cIcmoqp1MxjX8NlRkKGlQ md5 Nv48yAceNSh8z8 UcskU0EgGleETR6Oao82 2023-08-17 17:33:12 DzTjkKse
oVfV4tXeKpGM6X7UFMk7 6ObKC1Wo None .h5ad AnnData 10x reference pbmc68k None None 589484 eKVXV5okt5YRYjySMTKGEw md5 Nv48yAceNSh8z8 UcskU0EgGleETR6Oao82 2023-08-17 17:33:21 DzTjkKse 
ln.File.filter(labels__name="female").distinct().df()
storage_id key suffix accessor description version initial_version_id size hash hash_type transform_id run_id updated_at created_by_id
id
9SaWYjVX2DwR0dCC2l1B 6ObKC1Wo None .h5ad AnnData Detmar22 None None 17342743 lC0PrTQici8k6yWBFsaJzg md5 Nv48yAceNSh8z8 UcskU0EgGleETR6Oao82 2023-08-17 17:32:54 DzTjkKse

Intersect measured genes between two datasets#

file1 = ln.File.filter(description="Conde22").one()
file2 = ln.File.filter(description="10x reference pbmc68k").one()

file1.describe()

💡 File(id=P8fI3RY9UtTAEQfY7wx1, key=None, suffix=.h5ad, accessor=AnnData, description=Conde22, version=None, size=28061905, hash=3cIcmoqp1MxjX8NlRkKGlQ, hash_type=md5, created_at=2023-08-17 17:33:12.867793+00:00, updated_at=2023-08-17 17:33:12.867819+00:00)

Provenance:
    🗃️ storage: Storage(id='6ObKC1Wo', root='/home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna', type='local', updated_at=2023-08-17 17:33:27, created_by_id='DzTjkKse')
    📔 transform: Transform(id='Nv48yAceNSh8z8', name='Curate & link scRNA-seq datasets', short_name='scrna', stem_id='Nv48yAceNSh8', version='0', type='notebook', updated_at=2023-08-17 17:33:21, created_by_id='DzTjkKse')
    👣 run: Run(id='UcskU0EgGleETR6Oao82', run_at=2023-08-17 17:32:36, transform_id='Nv48yAceNSh8z8', created_by_id='DzTjkKse')
    👤 created_by: User(id='DzTjkKse', handle='testuser1', email='testuser1@lamin.ai', name='Test User1', updated_at=2023-08-17 17:33:27)
Features:
  var (X):
    🔗 index (36503, bionty.Gene.id): ['a1AGqGk4ywu1', 'dNgfeMyI0d1p', 'br4ZjTdWVCNF', '3Ucw70JttwA8', '9bqHaU58Tjf1'...]
  external:
    🔗 species (1, bionty.Species): ['human']
  obs (metadata):
    🔗 cell_type (32, bionty.CellType): ['macrophage', 'T follicular helper cell', 'plasma cell', 'regulatory T cell', 'animal cell']
    🔗 assay (3, bionty.ExperimentalFactor): ["10x 5' v2", "10x 5' v1", "10x 3' v3"]
    🔗 tissue (17, bionty.Tissue): ['liver', 'lamina propria', 'mesenteric lymph node', 'omentum', 'spleen']
    🔗 donor (12, core.Label): ['A37', 'A31', 'A52', '637C', 'A35']

file1.view_lineage()
https://d33wubrfki0l68.cloudfront.net/5117e145061c78f9e9fb6ea9b287629813807a91/44260/_images/0234a79c4d582d5020d0dfb47096954ff6b46f5a8308e6e5e1b0be8a92eb5875.svg
file2.describe()

💡 File(id=oVfV4tXeKpGM6X7UFMk7, key=None, suffix=.h5ad, accessor=AnnData, description=10x reference pbmc68k, version=None, size=589484, hash=eKVXV5okt5YRYjySMTKGEw, hash_type=md5, created_at=2023-08-17 17:33:21.798553+00:00, updated_at=2023-08-17 17:33:21.798576+00:00)

Provenance:
    🗃️ storage: Storage(id='6ObKC1Wo', root='/home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna', type='local', updated_at=2023-08-17 17:33:27, created_by_id='DzTjkKse')
    📔 transform: Transform(id='Nv48yAceNSh8z8', name='Curate & link scRNA-seq datasets', short_name='scrna', stem_id='Nv48yAceNSh8', version='0', type='notebook', updated_at=2023-08-17 17:33:21, created_by_id='DzTjkKse')
    👣 run: Run(id='UcskU0EgGleETR6Oao82', run_at=2023-08-17 17:32:36, transform_id='Nv48yAceNSh8z8', created_by_id='DzTjkKse')
    👤 created_by: User(id='DzTjkKse', handle='testuser1', email='testuser1@lamin.ai', name='Test User1', updated_at=2023-08-17 17:33:27)
Features:
  var (X):
    🔗 index (695, bionty.Gene.id): ['S40ynPlt1FR6', 'NlRk1RxzXPJS', '10ogBGiTBuv4', 'E231nvehewhz', 'Z2SBxpiWYpPs'...]
  obs (metadata):
    🔗 cell_type (9, bionty.CellType): ['CD8-positive, CD25-positive, alpha-beta regulatory T cell', 'cytotoxic T cell', 'effector memory CD4-positive, alpha-beta T cell, terminally differentiated', 'CD14-positive, CD16-negative classical monocyte', 'CD38-negative naive B cell']

file2.view_lineage()
https://d33wubrfki0l68.cloudfront.net/bc4b6176dcefa356db44620829538448919a810b/8ce96/_images/b24b62163f38771d3dce867e5c6eb3447ce4e35ee4a69dabd2cf56f4afc3801f.svg
file1_adata = file1.load()
file2_adata = file2.load()

💡 adding file P8fI3RY9UtTAEQfY7wx1 as input for run ZdrKUiFPGdQnnd9FGsEa, adding parent transform Nv48yAceNSh8z8

💡 adding file oVfV4tXeKpGM6X7UFMk7 as input for run ZdrKUiFPGdQnnd9FGsEa, adding parent transform Nv48yAceNSh8z8

file2_adata.obs.cell_type.head()

index
GCAGGGCTGGATTC-1                                       dendritic cell
CTTTAGTGGTTACG-6                                B cell, CD19-positive
TGACTGGAACCATG-7                                       dendritic cell
TCAATCACCCTTCG-8                                B cell, CD19-positive
CGTTATACAGTACC-8    effector memory CD4-positive, alpha-beta T cel...
Name: cell_type, dtype: category
Categories (9, object): ['CD8-positive, CD25-positive, alpha-beta regul..., 'effector memory CD4-positive, alpha-beta T ce..., 'cytotoxic T cell', 'CD38-negative naive B cell', ..., 'B cell, CD19-positive', 'conventional dendritic cell', 'CD16-positive, CD56-dim natural killer cell, ..., 'dendritic cell']

Here we compute shared genes without loading files:

file1_genes = file1.features["var"]
file2_genes = file2.features["var"]

shared_genes = file1_genes & file2_genes
shared_genes.list("symbol")[:10]

['PPP1R14A',
 'RSL1D1',
 'KLF6',
 'CD47',
 'BBX',
 'OTUB1',
 'GATA3',
 'DCXR',
 'SRRM2',
 'IL2RB']

We also need to convert the ensembl_gene_id to symbol for file2 so that they can be concatenated:

mapper = (
    pd.DataFrame(file2_genes.values_list("ensembl_gene_id", "symbol"))
    .drop_duplicates(0)
    .set_index(0)[1]
)
mapper.head()

0
ENSG00000158050    DUSP2
ENSG00000129351     ILF3
ENSG00000106244    PDAP1
ENSG00000105383     CD33
ENSG00000171840    NINJ2
Name: 1, dtype: object

file1_adata.var.rename(index=mapper, inplace=True)

Intersect cell types#

file1_celltypes = file1.cell_types.all()
file2_celltypes = file2.cell_types.all()

shared_celltypes = file1_celltypes & file2_celltypes
shared_celltypes_names = shared_celltypes.list("name")
shared_celltypes_names

['CD16-positive, CD56-dim natural killer cell, human',
 'conventional dendritic cell']

We can now subset the two datasets by shared cell types:

file1_adata_subset = file1_adata[
    file1_adata.obs["cell_type"].isin(shared_celltypes_names)
]
file1_adata_subset.obs["cell_type"].value_counts()

CD16-positive, CD56-dim natural killer cell, human    114
conventional dendritic cell                             7
Name: cell_type, dtype: int64

file2_adata_subset = file2_adata[
    file2_adata.obs["cell_type"].isin(shared_celltypes_names)
]
file2_adata_subset.obs["cell_type"].value_counts()

CD16-positive, CD56-dim natural killer cell, human    3
conventional dendritic cell                           2
Name: cell_type, dtype: int64

adata_concat = ad.concat(
    [file1_adata_subset, file2_adata_subset],
    label="file",
    keys=[file1.description, file2.description],
)
adata_concat

AnnData object with n_obs × n_vars = 126 × 695
    obs: 'cell_type', 'file'
    obsm: 'X_umap'

adata_concat.obs.value_counts()

cell_type                                           file                 
CD16-positive, CD56-dim natural killer cell, human  Conde22                  114
conventional dendritic cell                         Conde22                    7
CD16-positive, CD56-dim natural killer cell, human  10x reference pbmc68k      3
conventional dendritic cell                         10x reference pbmc68k      2
dtype: int64
Hide code cell content 
!lamin delete --force test-scrna
!rm -r ./test-scrna

💡 deleting instance testuser1/test-scrna
✅     deleted instance settings file: /home/runner/.lamin/instance--testuser1--test-scrna.env

✅     instance cache deleted
✅     deleted '.lndb' sqlite file
❗     consider manually deleting your stored data: /home/runner/work/lamin-usecases/lamin-usecases/docs/test-scrna