diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b271c2e..c34ffd8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,52 +6,24 @@ version: 2 jobs: build: docker: - # specify the version you desire here - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: circleci/python:3.6.1 - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 + - image: continuumio/miniconda3 working_directory: ~/repo steps: - checkout - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - - run: - name: install dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip3 install -q -r requirements.txt - pip3 install -q torch - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} - - run: name: run tests command: | - . venv/bin/activate - pip3 install . + conda install pytorch cpuonly -c pytorch-nightly -q -y + apt-get update + apt-get install -y gcc + apt-get install -y g++ + pip install . python test/test_nested_tensor_autograd.py python test/test_nested_tensor_class.py python test/test_nested_tensor_functional.py python test/test_nested_tensor_masking.py python test/test_nested_tensor_nary.py python test/test_nested_tensor_tensorwise.py - - - store_artifacts: - path: test-reports - destination: test-reports diff --git a/README.md b/README.md index ae7de137..aee43900 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # The nestedtensor package -NOTE: PLEASE NOTE, NESTEDTENSOR IS UNDER ACTIVE DEVELOPMENT AND EVERYTHING HERE IS SUBJECT TO CHANGE. +NOTE: nestedtensor is under active development and various aspects may change. +NOTE: We test and develop against nightlies! Please use the most recent version of PyTorch if you plan to use this code. ## Motivation @@ -44,7 +45,7 @@ Import nested tensors and torch via ```from nestedtensor import torch``` ### Creation ``` -nt = torch.nested_tensor( +nt = nestedtensor.nested_tensor( [ [ torch.rand(2, 3), @@ -62,13 +63,15 @@ b = torch.tensor([[2, 2], [3, 3], [4, 4], [5, 5]]) -nt2 = torch.nested_tensor([[a],[b]]) +nt2 = nestedtensor.nested_tensor([[a],[b]]) ``` The level of nesting is inferred from the input. The constructor always copies. Whatever you pass into the constructor will share no data with what the constructor returns. This matches torch.tensor's behavior. If given a NestedTensor or Tensor it will return a detached copy, which is consistent with the behavior of torch.tensor. Remember that you cannot mix Tensors and NestedTensors within a given list. +A side-note on naming: nestedtensor is a python packed and as such [shouldn't have underscores and is lower case](https://www.python.org/dev/peps/pep-0008/#package-and-module-names), but nested_tensor is a python function and as [such should use underscores](https://www.python.org/dev/peps/pep-0008/#function-and-variable-names) in contrast to the [CapWorded NestedTensor class](https://www.python.org/dev/peps/pep-0008/#class-names). + ### Conversion/unbind() A user can retrieve the constituent Tensors via unbind. Unbind is currently used by torch to turn Tensors into tuples of Tensors. Unbind always returns a tuple of views. @@ -80,7 +83,7 @@ A user can retrieve the constituent Tensors via unbind. Unbind is currently used ... [torch.rand(3, 2)] ... ] >>> ->>> b = torch.nested_tensor(a) +>>> b = nestedtensor.nested_tensor(a) >>> print(b) nested_tensor([ [ @@ -136,8 +139,8 @@ tensor([ 9, 11]) >>> print(simple_fn(c, d)) tensor([10, 10]) >>> ->>> n = torch.nested_tensor([a, c]) ->>> m = torch.nested_tensor([b, d]) +>>> n = nestedtensor.nested_tensor([a, c]) +>>> m = nestedtensor.nested_tensor([b, d]) >>> print(simple_fn(n, m)) nested_tensor([ tensor([ 9, 11]), diff --git a/examples/basic.ipynb b/examples/basic.ipynb index 89ca8f0c..fbcd810d 100644 --- a/examples/basic.ipynb +++ b/examples/basic.ipynb @@ -15,7 +15,8 @@ "metadata": {}, "outputs": [], "source": [ - "from nestedtensor import torch\n", + "import torch\n", + "import nestedtensor\n", "from IPython.display import Markdown, display\n", "\n", "def print_eval(s):\n", @@ -54,15 +55,15 @@ "text": [ "nested_tensor([\n", "\t[\n", - "\t\ttensor([[0.8264, 0.2200, 0.4197],\n", - "\t\t [0.6789, 0.7460, 0.1694]]),\n", - "\t\ttensor([[0.7467, 0.8433, 0.6429, 0.9890, 0.0170],\n", - "\t\t [0.6297, 0.3899, 0.7025, 0.0812, 0.9585],\n", - "\t\t [0.1113, 0.4260, 0.4245, 0.7971, 0.7910],\n", - "\t\t [0.7077, 0.6765, 0.0228, 0.5461, 0.4095]])\n", + "\t\ttensor([[0.1525, 0.9457, 0.8438],\n", + "\t\t [0.6784, 0.9376, 0.5344]]),\n", + "\t\ttensor([[0.5654, 0.6054, 0.2726, 0.8868, 0.3417],\n", + "\t\t [0.1225, 0.4104, 0.9022, 0.6978, 0.2081],\n", + "\t\t [0.5641, 0.2983, 0.7589, 0.5495, 0.1304],\n", + "\t\t [0.1999, 0.3803, 0.0336, 0.4855, 0.9838]])\n", "\t],\n", "\t[\n", - "\t\ttensor([[0.0660, 0.9756]])\n", + "\t\ttensor([[0.8105, 0.6778]])\n", "\t]\n", "])\n", "\n" @@ -70,7 +71,7 @@ } ], "source": [ - "nt = torch.nested_tensor(\n", + "nt = nestedtensor.nested_tensor(\n", " [\n", " [\n", " torch.rand(2, 3),\n", @@ -227,7 +228,7 @@ " [3, 3],\n", " [4, 4],\n", " [5, 5]])\n", - "nt2 = torch.nested_tensor([[a],[b]])\n", + "nt2 = nestedtensor.nested_tensor([[a],[b]])\n", "print_eval(\"nt2.nested_dim()\")\n", "print_eval(\"nt2.tensor_dim()\")\n", "print_eval(\"nt2.dim()\")" @@ -281,15 +282,15 @@ "text": [ "nested_tensor([\n", "\t[\n", - "\t\ttensor([[0.8264, 0.2200, 0.4197],\n", - "\t\t [0.6789, 0.7460, 0.1694]]),\n", - "\t\ttensor([[0.7467, 0.8433, 0.6429, 0.9890, 0.0170],\n", - "\t\t [0.6297, 0.3899, 0.7025, 0.0812, 0.9585],\n", - "\t\t [0.1113, 0.4260, 0.4245, 0.7971, 0.7910],\n", - "\t\t [0.7077, 0.6765, 0.0228, 0.5461, 0.4095]])\n", + "\t\ttensor([[0.1525, 0.9457, 0.8438],\n", + "\t\t [0.6784, 0.9376, 0.5344]]),\n", + "\t\ttensor([[0.5654, 0.6054, 0.2726, 0.8868, 0.3417],\n", + "\t\t [0.1225, 0.4104, 0.9022, 0.6978, 0.2081],\n", + "\t\t [0.5641, 0.2983, 0.7589, 0.5495, 0.1304],\n", + "\t\t [0.1999, 0.3803, 0.0336, 0.4855, 0.9838]])\n", "\t],\n", "\t[\n", - "\t\ttensor([[0.0660, 0.9756]])\n", + "\t\ttensor([[0.8105, 0.6778]])\n", "\t]\n", "])\n", "\n" @@ -621,7 +622,7 @@ { "data": { "text/markdown": [ - "**$ torch.nested_tensor_from_tensor_mask(tensor, mask)**" + "**$ nestedtensor.nested_tensor_from_tensor_mask(tensor, mask)**" ], "text/plain": [ "" @@ -646,7 +647,7 @@ { "data": { "text/markdown": [ - "**$ torch.nested_tensor_from_padded_tensor(tensor, padding=0)**" + "**$ nestedtensor.nested_tensor_from_padded_tensor(tensor, padding=0)**" ], "text/plain": [ "" @@ -688,9 +689,9 @@ " [ True, True, True, True]]])\n", "print_eval(\"tensor\")\n", "print_eval(\"mask\")\n", - "nt2 = torch.nested_tensor_from_tensor_mask(tensor, mask)\n", - "print_eval(\"torch.nested_tensor_from_tensor_mask(tensor, mask)\")\n", - "print_eval(\"torch.nested_tensor_from_padded_tensor(tensor, padding=0)\")" + "nt2 = nestedtensor.nested_tensor_from_tensor_mask(tensor, mask)\n", + "print_eval(\"nestedtensor.nested_tensor_from_tensor_mask(tensor, mask)\")\n", + "print_eval(\"nestedtensor.nested_tensor_from_padded_tensor(tensor, padding=0)\")" ] }, { @@ -795,12 +796,12 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([[0.8264, 0.2200, 0.4197],\n", - "\t [0.6789, 0.7460, 0.1694]]),\n", - "\ttensor([[0.7467, 0.8433, 0.6429, 0.9890, 0.0170],\n", - "\t [0.6297, 0.3899, 0.7025, 0.0812, 0.9585],\n", - "\t [0.1113, 0.4260, 0.4245, 0.7971, 0.7910],\n", - "\t [0.7077, 0.6765, 0.0228, 0.5461, 0.4095]])\n", + "\ttensor([[0.1525, 0.9457, 0.8438],\n", + "\t [0.6784, 0.9376, 0.5344]]),\n", + "\ttensor([[0.5654, 0.6054, 0.2726, 0.8868, 0.3417],\n", + "\t [0.1225, 0.4104, 0.9022, 0.6978, 0.2081],\n", + "\t [0.5641, 0.2983, 0.7589, 0.5495, 0.1304],\n", + "\t [0.1999, 0.3803, 0.0336, 0.4855, 0.9838]])\n", "])\n", "\n" ] @@ -822,7 +823,7 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([[0.0660, 0.9756]])\n", + "\ttensor([[0.8105, 0.6778]])\n", "])\n", "\n" ] @@ -857,15 +858,15 @@ "text": [ "nested_tensor([\n", "\t[\n", - "\t\ttensor([[0.6776, 0.9759, 0.9132],\n", - "\t\t [0.7783, 0.7344, 0.9857]]),\n", - "\t\ttensor([[0.7467, 0.8433, 0.6429, 0.9890, 0.0170],\n", - "\t\t [0.6297, 0.3899, 0.7025, 0.0812, 0.9585],\n", - "\t\t [0.1113, 0.4260, 0.4245, 0.7971, 0.7910],\n", - "\t\t [0.7077, 0.6765, 0.0228, 0.5461, 0.4095]])\n", + "\t\ttensor([[0.9884, 0.5852, 0.6646],\n", + "\t\t [0.7786, 0.5917, 0.8606]]),\n", + "\t\ttensor([[0.5654, 0.6054, 0.2726, 0.8868, 0.3417],\n", + "\t\t [0.1225, 0.4104, 0.9022, 0.6978, 0.2081],\n", + "\t\t [0.5641, 0.2983, 0.7589, 0.5495, 0.1304],\n", + "\t\t [0.1999, 0.3803, 0.0336, 0.4855, 0.9838]])\n", "\t],\n", "\t[\n", - "\t\ttensor([[0.0660, 0.9756]])\n", + "\t\ttensor([[0.8105, 0.6778]])\n", "\t]\n", "])\n", "\n" diff --git a/examples/indexing.ipynb b/examples/indexing.ipynb index 0e70a6be..69adff3d 100644 --- a/examples/indexing.ipynb +++ b/examples/indexing.ipynb @@ -13,7 +13,8 @@ "metadata": {}, "outputs": [], "source": [ - "from nestedtensor import torch\n", + "import torch\n", + "import nestedtensor\n", "from IPython.display import Markdown, display\n", "\n", "def print_eval(s):\n", @@ -62,7 +63,7 @@ } ], "source": [ - "nt2 = torch.nested_tensor(\n", + "nt2 = nestedtensor.nested_tensor(\n", "[\n", " [\n", " torch.tensor([[1.0, 0.5], [0.1, 0.6]]),\n", @@ -393,8 +394,8 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([1.0000, 0.5000]),\n", - "\ttensor([5.5000, 3.3000])\n", + " tensor([1.0000, 0.5000]),\n", + " tensor([5.5000, 3.3000]),\n", "])\n", "\n" ] @@ -416,10 +417,10 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([[1.0000, 0.5000],\n", - "\t [0.1000, 0.6000]]),\n", - "\ttensor([[3.0000, 1.0000],\n", - "\t [0.5000, 0.7000]])\n", + " tensor([[1.0000, 0.5000],\n", + " [0.1000, 0.6000]]),\n", + " tensor([[3.0000, 1.0000],\n", + " [0.5000, 0.7000]]),\n", "])\n", "\n" ] @@ -441,10 +442,10 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([[1.0000, 0.5000],\n", - "\t [0.1000, 0.6000]]),\n", - "\ttensor([[5.5000, 3.3000],\n", - "\t [2.2000, 6.6000]])\n", + " tensor([[1.0000, 0.5000],\n", + " [0.1000, 0.6000]]),\n", + " tensor([[5.5000, 3.3000],\n", + " [2.2000, 6.6000]]),\n", "])\n", "\n" ] @@ -513,18 +514,18 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\t[\n", - "\t\ttensor([[0.1000, 0.6000],\n", - "\t\t [1.0000, 0.5000]]),\n", - "\t\ttensor([[2.2000, 6.6000],\n", - "\t\t [5.5000, 3.3000]])\n", - "\t],\n", - "\t[\n", - "\t\ttensor([[0.5000, 0.7000],\n", - "\t\t [3.0000, 1.0000]]),\n", - "\t\ttensor([[1., 2.],\n", - "\t\t [5., 4.]])\n", - "\t]\n", + " nested_tensor([\n", + " tensor([[0.1000, 0.6000],\n", + " [1.0000, 0.5000]]),\n", + " tensor([[2.2000, 6.6000],\n", + " [5.5000, 3.3000]]),\n", + "]),\n", + " nested_tensor([\n", + " tensor([[0.5000, 0.7000],\n", + " [3.0000, 1.0000]]),\n", + " tensor([[1., 2.],\n", + " [5., 4.]]),\n", + "]),\n", "])\n", "\n" ] @@ -612,30 +613,30 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\t[\n", - "\t\ttensor([[[0.1000, 0.6000],\n", - "\t\t [1.0000, 0.5000]],\n", - "\t\t\n", - "\t\t [[1.0000, 0.5000],\n", - "\t\t [0.1000, 0.6000]]]),\n", - "\t\ttensor([[[2.2000, 6.6000],\n", - "\t\t [5.5000, 3.3000]],\n", - "\t\t\n", - "\t\t [[5.5000, 3.3000],\n", - "\t\t [2.2000, 6.6000]]])\n", - "\t],\n", - "\t[\n", - "\t\ttensor([[[0.5000, 0.7000],\n", - "\t\t [3.0000, 1.0000]],\n", - "\t\t\n", - "\t\t [[3.0000, 1.0000],\n", - "\t\t [0.5000, 0.7000]]]),\n", - "\t\ttensor([[[1., 2.],\n", - "\t\t [5., 4.]],\n", - "\t\t\n", - "\t\t [[5., 4.],\n", - "\t\t [1., 2.]]])\n", - "\t]\n", + " nested_tensor([\n", + " tensor([[[0.1000, 0.6000],\n", + " [1.0000, 0.5000]],\n", + "\n", + " [[1.0000, 0.5000],\n", + " [0.1000, 0.6000]]]),\n", + " tensor([[[2.2000, 6.6000],\n", + " [5.5000, 3.3000]],\n", + "\n", + " [[5.5000, 3.3000],\n", + " [2.2000, 6.6000]]]),\n", + "]),\n", + " nested_tensor([\n", + " tensor([[[0.5000, 0.7000],\n", + " [3.0000, 1.0000]],\n", + "\n", + " [[3.0000, 1.0000],\n", + " [0.5000, 0.7000]]]),\n", + " tensor([[[1., 2.],\n", + " [5., 4.]],\n", + "\n", + " [[5., 4.],\n", + " [1., 2.]]]),\n", + "]),\n", "])\n", "\n" ] @@ -704,14 +705,14 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\t[\n", - "\t\ttensor([1.0000, 0.1000]),\n", - "\t\ttensor([5.5000, 2.2000])\n", - "\t],\n", - "\t[\n", - "\t\ttensor([3.0000, 0.5000]),\n", - "\t\ttensor([5., 1.])\n", - "\t]\n", + " nested_tensor([\n", + " tensor([1.0000, 0.1000]),\n", + " tensor([5.5000, 2.2000]),\n", + "]),\n", + " nested_tensor([\n", + " tensor([3.0000, 0.5000]),\n", + " tensor([5., 1.]),\n", + "]),\n", "])\n", "\n", "$ nt2[..., 0]\n", diff --git a/examples/layers.ipynb b/examples/layers.ipynb index c1da3509..545412a5 100644 --- a/examples/layers.ipynb +++ b/examples/layers.ipynb @@ -6,7 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "from nestedtensor import torch\n", + "import torch\n", + "import nestedtensor\n", "from IPython.display import Markdown, display\n", "\n", "def print_eval(s):\n", @@ -52,12 +53,12 @@ } ], "source": [ - "nt = torch.nested_tensor([\n", + "nt = nestedtensor.nested_tensor([\n", " torch.rand(3, 10, 30),\n", " torch.rand(3, 20, 40),\n", " torch.rand(3, 30, 50)\n", "])\n", - "nt1 = torch.nested_tensor([\n", + "nt1 = nestedtensor.nested_tensor([\n", " torch.rand(1, 3, 10, 30),\n", " torch.rand(1, 3, 20, 40),\n", " torch.rand(1, 3, 30, 50)\n", @@ -81,7 +82,7 @@ { "data": { "text/markdown": [ - "**$ torch.tensorwise()(torch.nn.functional.conv2d)(nt, weight)**" + "**$ nestedtensor.tensorwise()(torch.nn.functional.conv2d)(nt, weight)**" ], "text/plain": [ "" @@ -113,7 +114,7 @@ { "data": { "text/markdown": [ - "**$ torch.tensorwise()(torch.nn.functional.conv2d)(nt1, weight).size()**" + "**$ nestedtensor.tensorwise()(torch.nn.functional.conv2d)(nt1, weight).size()**" ], "text/plain": [ "" @@ -133,11 +134,11 @@ ], "source": [ "try:\n", - " print_eval(\"torch.tensorwise()(torch.nn.functional.conv2d)(nt, weight)\")\n", + " print_eval(\"nestedtensor.tensorwise()(torch.nn.functional.conv2d)(nt, weight)\")\n", "except RuntimeError as e:\n", " print_eval(\"str(e)\")\n", " \n", - "print_eval(\"torch.tensorwise()(torch.nn.functional.conv2d)(nt1, weight).size()\")" + "print_eval(\"nestedtensor.tensorwise()(torch.nn.functional.conv2d)(nt1, weight).size()\")" ] }, { @@ -190,17 +191,17 @@ "metadata": {}, "outputs": [], "source": [ - "nt2 = (torch.nested_tensor([\n", + "nt2 = (nestedtensor.nested_tensor([\n", " torch.rand(1, 30),\n", " torch.rand(1, 40),\n", " torch.rand(1, 50)\n", "]) * 10).to(torch.int64)\n", - "nt3 = (torch.nested_tensor([\n", + "nt3 = (nestedtensor.nested_tensor([\n", " torch.rand(30),\n", " torch.rand(40),\n", " torch.rand(50)\n", "]) * 10).to(torch.int64)\n", - "nt4 = (torch.nested_tensor([\n", + "nt4 = (nestedtensor.nested_tensor([\n", " [\n", " torch.rand(1, 30),\n", " ],\n", @@ -215,168 +216,16 @@ "cell_type": "code", "execution_count": 6, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "**$ torch.nn.functional.embedding_bag(nt2, weight).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\ttorch.Size([1, 256]),\n", - "\ttorch.Size([1, 256]),\n", - "\ttorch.Size([1, 256])\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ torch.nn.functional.embedding_bag(nt3, weight).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\ttorch.Size([256]),\n", - "\ttorch.Size([256]),\n", - "\ttorch.Size([256])\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ torch.nn.functional.embedding_bag(nt4, weight).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\t(\n", - "\t\ttorch.Size([1, 256])\n", - "\t),\n", - "\t(\n", - "\t\ttorch.Size([1, 256]),\n", - "\t\ttorch.Size([1, 256])\n", - "\t)\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ torch.nn.EmbeddingBag(100, 256)(nt2).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\ttorch.Size([1, 256]),\n", - "\ttorch.Size([1, 256]),\n", - "\ttorch.Size([1, 256])\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ torch.nn.EmbeddingBag(100, 256)(nt3).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\ttorch.Size([256]),\n", - "\ttorch.Size([256]),\n", - "\ttorch.Size([256])\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ torch.nn.EmbeddingBag(100, 256)(nt4).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\t(\n", - "\t\ttorch.Size([1, 256])\n", - "\t),\n", - "\t(\n", - "\t\ttorch.Size([1, 256]),\n", - "\t\ttorch.Size([1, 256])\n", - "\t)\n", - "))\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "weight = torch.rand(100, 256)\n", - "print_eval(\"torch.nn.functional.embedding_bag(nt2, weight).nested_size()\")\n", - "print_eval(\"torch.nn.functional.embedding_bag(nt3, weight).nested_size()\")\n", - "print_eval(\"torch.nn.functional.embedding_bag(nt4, weight).nested_size()\")\n", - "print_eval(\"torch.nn.EmbeddingBag(100, 256)(nt2).nested_size()\")\n", - "print_eval(\"torch.nn.EmbeddingBag(100, 256)(nt3).nested_size()\")\n", - "print_eval(\"torch.nn.EmbeddingBag(100, 256)(nt4).nested_size()\")" + "# THIS IS TEMPORARILY DISABLED\n", + "# weight = torch.rand(100, 256)\n", + "# print_eval(\"torch.nn.functional.embedding_bag(nt2, weight).nested_size()\")\n", + "# print_eval(\"torch.nn.functional.embedding_bag(nt3, weight).nested_size()\")\n", + "# print_eval(\"torch.nn.functional.embedding_bag(nt4, weight).nested_size()\")\n", + "# print_eval(\"torch.nn.EmbeddingBag(100, 256)(nt2).nested_size()\")\n", + "# print_eval(\"torch.nn.EmbeddingBag(100, 256)(nt3).nested_size()\")\n", + "# print_eval(\"torch.nn.EmbeddingBag(100, 256)(nt4).nested_size()\")" ] }, { @@ -401,14 +250,14 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([3., 9., 4., 9., 7., 5., 0., 4., 4., 6., 7., 4., 4., 9., 0., 4., 9., 2.,\n", - "\t 4., 4., 7., 3., 1., 7., 4., 1., 7., 1., 8., 9.]),\n", - "\ttensor([5., 2., 6., 3., 9., 2., 9., 6., 7., 8., 0., 4., 5., 7., 3., 0., 5., 4.,\n", - "\t 2., 5., 0., 8., 1., 8., 8., 0., 4., 8., 5., 8., 3., 2., 6., 8., 0., 3.,\n", - "\t 8., 3., 7., 5.]),\n", - "\ttensor([2., 0., 5., 1., 1., 7., 9., 3., 7., 1., 6., 2., 2., 4., 5., 2., 5., 2.,\n", - "\t 4., 6., 7., 1., 0., 2., 4., 7., 8., 2., 1., 1., 4., 7., 1., 4., 9., 6.,\n", - "\t 1., 6., 0., 2., 1., 6., 8., 7., 1., 5., 0., 4., 7., 0.])\n", + "\ttensor([0., 1., 8., 3., 0., 9., 0., 9., 6., 4., 6., 1., 0., 4., 9., 5., 7., 8.,\n", + "\t 1., 8., 2., 1., 5., 2., 4., 9., 4., 4., 6., 5.]),\n", + "\ttensor([1., 0., 6., 8., 9., 7., 0., 4., 0., 1., 3., 9., 6., 5., 2., 7., 2., 5.,\n", + "\t 9., 3., 2., 6., 4., 4., 0., 4., 2., 2., 5., 5., 8., 1., 1., 2., 3., 7.,\n", + "\t 3., 3., 6., 7.]),\n", + "\ttensor([6., 5., 0., 4., 3., 4., 8., 0., 7., 5., 7., 6., 4., 7., 2., 9., 1., 0.,\n", + "\t 3., 5., 3., 2., 5., 1., 8., 2., 1., 7., 0., 4., 8., 9., 2., 2., 6., 7.,\n", + "\t 9., 4., 2., 9., 6., 3., 2., 2., 4., 6., 7., 6., 8., 4.])\n", "])\n", "\n" ] @@ -460,7 +309,7 @@ { "data": { "text/markdown": [ - "**$ torch.nested_tensor(nt3.nested_size(1))**" + "**$ nestedtensor.nested_tensor(nt3.nested_size(1))**" ], "text/plain": [ "" @@ -498,21 +347,21 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([0.1000, 0.3000, 0.1333, 0.3000, 0.2333, 0.1667, 0.0000, 0.1333, 0.1333,\n", - "\t 0.2000, 0.2333, 0.1333, 0.1333, 0.3000, 0.0000, 0.1333, 0.3000, 0.0667,\n", - "\t 0.1333, 0.1333, 0.2333, 0.1000, 0.0333, 0.2333, 0.1333, 0.0333, 0.2333,\n", - "\t 0.0333, 0.2667, 0.3000]),\n", - "\ttensor([0.1250, 0.0500, 0.1500, 0.0750, 0.2250, 0.0500, 0.2250, 0.1500, 0.1750,\n", - "\t 0.2000, 0.0000, 0.1000, 0.1250, 0.1750, 0.0750, 0.0000, 0.1250, 0.1000,\n", - "\t 0.0500, 0.1250, 0.0000, 0.2000, 0.0250, 0.2000, 0.2000, 0.0000, 0.1000,\n", - "\t 0.2000, 0.1250, 0.2000, 0.0750, 0.0500, 0.1500, 0.2000, 0.0000, 0.0750,\n", - "\t 0.2000, 0.0750, 0.1750, 0.1250]),\n", - "\ttensor([0.0400, 0.0000, 0.1000, 0.0200, 0.0200, 0.1400, 0.1800, 0.0600, 0.1400,\n", - "\t 0.0200, 0.1200, 0.0400, 0.0400, 0.0800, 0.1000, 0.0400, 0.1000, 0.0400,\n", - "\t 0.0800, 0.1200, 0.1400, 0.0200, 0.0000, 0.0400, 0.0800, 0.1400, 0.1600,\n", - "\t 0.0400, 0.0200, 0.0200, 0.0800, 0.1400, 0.0200, 0.0800, 0.1800, 0.1200,\n", - "\t 0.0200, 0.1200, 0.0000, 0.0400, 0.0200, 0.1200, 0.1600, 0.1400, 0.0200,\n", - "\t 0.1000, 0.0000, 0.0800, 0.1400, 0.0000])\n", + "\ttensor([0.0000, 0.0333, 0.2667, 0.1000, 0.0000, 0.3000, 0.0000, 0.3000, 0.2000,\n", + "\t 0.1333, 0.2000, 0.0333, 0.0000, 0.1333, 0.3000, 0.1667, 0.2333, 0.2667,\n", + "\t 0.0333, 0.2667, 0.0667, 0.0333, 0.1667, 0.0667, 0.1333, 0.3000, 0.1333,\n", + "\t 0.1333, 0.2000, 0.1667]),\n", + "\ttensor([0.0250, 0.0000, 0.1500, 0.2000, 0.2250, 0.1750, 0.0000, 0.1000, 0.0000,\n", + "\t 0.0250, 0.0750, 0.2250, 0.1500, 0.1250, 0.0500, 0.1750, 0.0500, 0.1250,\n", + "\t 0.2250, 0.0750, 0.0500, 0.1500, 0.1000, 0.1000, 0.0000, 0.1000, 0.0500,\n", + "\t 0.0500, 0.1250, 0.1250, 0.2000, 0.0250, 0.0250, 0.0500, 0.0750, 0.1750,\n", + "\t 0.0750, 0.0750, 0.1500, 0.1750]),\n", + "\ttensor([0.1200, 0.1000, 0.0000, 0.0800, 0.0600, 0.0800, 0.1600, 0.0000, 0.1400,\n", + "\t 0.1000, 0.1400, 0.1200, 0.0800, 0.1400, 0.0400, 0.1800, 0.0200, 0.0000,\n", + "\t 0.0600, 0.1000, 0.0600, 0.0400, 0.1000, 0.0200, 0.1600, 0.0400, 0.0200,\n", + "\t 0.1400, 0.0000, 0.0800, 0.1600, 0.1800, 0.0400, 0.0400, 0.1200, 0.1400,\n", + "\t 0.1800, 0.0800, 0.0400, 0.1800, 0.1200, 0.0600, 0.0400, 0.0400, 0.0800,\n", + "\t 0.1200, 0.1400, 0.1200, 0.1600, 0.0800])\n", "])\n", "\n" ] @@ -543,8 +392,8 @@ "print_eval(\"nt3\")\n", "print_eval(\"nt3.size()\")\n", "print_eval(\"nt3.nested_size()\")\n", - "print_eval(\"torch.nested_tensor(nt3.nested_size(1))\")\n", - "nt4 = nt3 / torch.nested_tensor(nt3.nested_size(1))\n", + "print_eval(\"nestedtensor.nested_tensor(nt3.nested_size(1))\")\n", + "nt4 = nt3 / nestedtensor.nested_tensor(nt3.nested_size(1))\n", "print_eval(\"nt4\")\n", "print_eval(\"nt4.size()\")" ] @@ -604,7 +453,7 @@ } ], "source": [ - "nt5 = torch.nested_tensor([\n", + "nt5 = nestedtensor.nested_tensor([\n", " torch.rand(30, 10),\n", " torch.rand(40, 10),\n", " torch.rand(50, 10)\n", @@ -635,9 +484,9 @@ "output_type": "stream", "text": [ "nested_tensor([\n", - "\ttensor([21, 0, 24, 12, 16, 10, 21, 20, 5, 20]),\n", - "\ttensor([ 4, 1, 31, 2, 15, 25, 0, 2, 32, 19]),\n", - "\ttensor([42, 6, 14, 34, 42, 11, 48, 13, 36, 41])\n", + "\ttensor([28, 28, 18, 6, 17, 1, 17, 23, 18, 18]),\n", + "\ttensor([ 3, 23, 2, 4, 1, 31, 7, 14, 1, 0]),\n", + "\ttensor([38, 1, 47, 34, 46, 48, 44, 9, 11, 47])\n", "])\n", "\n" ] @@ -678,9 +527,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "tensor([[21, 0, 24, 12, 16, 10, 21, 20, 5, 20],\n", - " [ 4, 1, 31, 2, 15, 25, 0, 2, 32, 19],\n", - " [42, 6, 14, 34, 42, 11, 48, 13, 36, 41]])\n", + "tensor([[28, 28, 18, 6, 17, 1, 17, 23, 18, 18],\n", + " [ 3, 23, 2, 4, 1, 31, 7, 14, 1, 0],\n", + " [38, 1, 47, 34, 46, 48, 44, 9, 11, 47]])\n", "\n" ] } @@ -695,84 +544,12 @@ "cell_type": "code", "execution_count": 10, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "**$ nt5.nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\ttorch.Size([30, 10]),\n", - "\ttorch.Size([40, 10]),\n", - "\ttorch.Size([50, 10])\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ nt5.argmax(2).nested_size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.NestedSize((\n", - "\ttorch.Size([30]),\n", - "\ttorch.Size([40]),\n", - "\ttorch.Size([50])\n", - "))\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ torch.nn.functional.cross_entropy(nt5, nt5.argmax(2))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "nested_tensor([\n", - "\ttensor(1.9496),\n", - "\ttensor(1.9408),\n", - "\ttensor(1.9305)\n", - "])\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "print_eval(\"nt5.nested_size()\")\n", - "print_eval(\"nt5.argmax(2).nested_size()\")\n", - "print_eval(\"torch.nn.functional.cross_entropy(nt5, nt5.argmax(2))\")" + "# THIS IS TEMOPORARILY DISABLED\n", + "# print_eval(\"nt5.nested_size()\")\n", + "# print_eval(\"nt5.argmax(2).nested_size()\")\n", + "# print_eval(\"torch.nn.functional.cross_entropy(nt5, nt5.argmax(2))\")" ] }, { @@ -822,7 +599,7 @@ } ], "source": [ - "nt6 = torch.nested_tensor([torch.rand(10, 10), torch.rand(20, 20), torch.rand(30, 30)])\n", + "nt6 = nestedtensor.nested_tensor([torch.rand(10, 10), torch.rand(20, 20), torch.rand(30, 30)])\n", "print_eval(\"nt6.lu()[0].size()\")\n", "print_eval(\"nt6.lu()[1].size()\")" ] @@ -850,14 +627,14 @@ "text": [ "nested_tensor([\n", "\t[\n", - "\t\ttensor([[3.0984]]),\n", - "\t\ttensor([[5.3822, 5.3800],\n", - "\t\t [4.8560, 5.2896]])\n", + "\t\ttensor([[3.3967]]),\n", + "\t\ttensor([[3.2799, 2.8154],\n", + "\t\t [3.7403, 4.0024]])\n", "\t],\n", "\t[\n", - "\t\ttensor([[9.7878, 5.3839, 7.7967],\n", - "\t\t [8.1811, 6.1959, 7.1056],\n", - "\t\t [8.6106, 5.1237, 6.6942]])\n", + "\t\ttensor([[8.2538, 7.9232, 8.3564],\n", + "\t\t [7.1505, 6.9339, 8.7236],\n", + "\t\t [7.4973, 7.2823, 8.4991]])\n", "\t]\n", "])\n", "\n" @@ -865,8 +642,8 @@ } ], "source": [ - "nt7 = torch.nested_tensor([[torch.rand(1, 10), torch.rand(2, 20)], [torch.rand(3, 30)]])\n", - "nt8 = torch.nested_tensor([[torch.rand(10, 1), torch.rand(20, 2)], [torch.rand(30, 3)]])\n", + "nt7 = nestedtensor.nested_tensor([[torch.rand(1, 10), torch.rand(2, 20)], [torch.rand(3, 30)]])\n", + "nt8 = nestedtensor.nested_tensor([[torch.rand(10, 1), torch.rand(20, 2)], [torch.rand(30, 3)]])\n", "print_eval(\"torch.mm(nt7, nt8)\")" ] }, diff --git a/examples/models.ipynb b/examples/models.ipynb index 5de4e1d7..072f428f 100644 --- a/examples/models.ipynb +++ b/examples/models.ipynb @@ -6,7 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "from nestedtensor import torch\n", + "import torch\n", + "import nestedtensor\n", "from IPython.display import Markdown, display\n", "def print_eval(s):\n", " colorS = \"$ {}\".format(s)\n", @@ -37,48 +38,7 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "**$ time(lambda: model(text, offsets))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average 0.3052ms based on 32763 samples\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ time(lambda: model(nt_text, None))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average 0.4228ms based on 23652 samples\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "def generate_tensors(num_tensor, vocab_size):\n", " sentence_lengths = torch.normal(75.0, 10.0, size=(num_tensor,)).long()\n", @@ -106,15 +66,16 @@ " def forward(self, text, offsets):\n", " emb = self.embedding(text, offsets)\n", " return self.fc(emb)\n", - " \n", - "vocab_size = 10000\n", - "model = TextSentiment(10000, 256, 5)\n", - "tensors = generate_tensors(16, 10000)\n", - "text, offsets = generate_text(tensors)\n", - "nt_text = torch.nested_tensor(tensors)\n", "\n", - "print_eval(\"time(lambda: model(text, offsets))\")\n", - "print_eval(\"time(lambda: model(nt_text, None))\")" + "# THIS IS TEMPORARILY DISABLED\n", + "# vocab_size = 10000\n", + "# model = TextSentiment(10000, 256, 5)\n", + "# tensors = generate_tensors(16, 10000)\n", + "# text, offsets = generate_text(tensors)\n", + "# nt_text = nestedtensor.nested_tensor(tensors)\n", + "\n", + "# print_eval(\"time(lambda: model(text, offsets))\")\n", + "# print_eval(\"time(lambda: model(nt_text, None))\")" ] }, { @@ -158,87 +119,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "average 75.3909ms based on 133 samples\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ time(lambda: model(nested_images))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average 801.7393ms based on 13 samples\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ nested_irregular_images.numel()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "692112\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ nested_irregular_images.size()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(64, 3, None, None)\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ time(lambda: model(nested_irregular_images))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average 2866.2733ms based on 4 samples\n", + "average 55.8146ms based on 180 samples\n", "\n" ] } @@ -251,140 +132,42 @@ "print_eval(\"images.numel()\")\n", "print_eval(\"time(lambda: model(images))\")\n", "\n", - "nested_images = torch.nested_tensor(torch.rand(128, 3, 40, 50).unbind())\n", - "print_eval(\"time(lambda: model(nested_images))\")\n", + "# THIS IS TEMPORARILY DISABLED\n", + "# nested_images = nestedtensor.nested_tensor(torch.rand(128, 3, 40, 50).unbind())\n", + "# print_eval(\"time(lambda: model(nested_images))\")\n", "\n", - "# There is still about a 10x gap in performance, which however\n", - "# can be significantly allieviated via custom code (e.g. using im2col).\n", - "images = [torch.rand(3, (i * 16) % 40 + 40, (i * 16) % 50 + 40) for i in range(64)]\n", - "nested_irregular_images = torch.nested_tensor(images)\n", - "print_eval(\"nested_irregular_images.numel()\")\n", - "print_eval(\"nested_irregular_images.size()\")\n", - "print_eval(\"time(lambda: model(nested_irregular_images))\")" + "# # There is still about a 10x gap in performance, which however\n", + "# # can be significantly allieviated via custom code (e.g. using im2col).\n", + "# images = [torch.rand(3, (i * 16) % 40 + 40, (i * 16) % 50 + 40) for i in range(64)]\n", + "# nested_irregular_images = nestedtensor.nested_tensor(images)\n", + "# print_eval(\"nested_irregular_images.numel()\")\n", + "# print_eval(\"nested_irregular_images.size()\")\n", + "# print_eval(\"time(lambda: model(nested_irregular_images))\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "**$ nt_text.nested_size(1)**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(74, 59, 68, 67, 85, 81, 75, 68, 74, 69, 80, 70, 78, 75, 91, 89, 59, 61, 75, 60, 73, 75, 71, 66, 73, 55, 78, 59, 75, 55, 70, 87)\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ nt_text.numel()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "587520\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ text.numel()**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "614400\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ time(lambda: torch.nn.LSTM(256, 512, 6, batch_first=True)(nt_text, (h0, c0)))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average 2431.1809ms based on 5 samples\n", - "\n" - ] - }, - { - "data": { - "text/markdown": [ - "**$ time(lambda: torch.nn.LSTM(256, 512, 6, batch_first=True)(text, (h0, c0)))**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average 518.9265ms based on 20 samples\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "def generate_tensors(num_tensor, num_features):\n", - " sentence_lengths = torch.normal(75.0, 10.0, size=(num_tensor,)).long()\n", - " return [torch.rand(l.item(), num_features) for l in sentence_lengths]\n", + "# THIS IS TEMPORARILY DISABLED\n", + "\n", + "# def generate_tensors(num_tensor, num_features):\n", + "# sentence_lengths = torch.normal(75.0, 10.0, size=(num_tensor,)).long()\n", + "# return [torch.rand(l.item(), num_features) for l in sentence_lengths]\n", "\n", - "tensors = generate_tensors(32, 256)\n", - "nt_text = torch.nested_tensor(tensors)\n", - "text = torch.rand(32, 75, 256)\n", + "# tensors = generate_tensors(32, 256)\n", + "# nt_text = nestedtensor.nested_tensor(tensors)\n", + "# text = torch.rand(32, 75, 256)\n", "\n", - "h0 = torch.randn(6, len(nt_text), 512)\n", - "c0 = torch.randn(6, len(nt_text), 512)\n", - "print_eval(\"nt_text.nested_size(1)\")\n", - "print_eval(\"nt_text.numel()\")\n", - "print_eval(\"text.numel()\")\n", - "print_eval(\"time(lambda: torch.nn.LSTM(256, 512, 6, batch_first=True)(nt_text, (h0, c0)))\")\n", - "print_eval(\"time(lambda: torch.nn.LSTM(256, 512, 6, batch_first=True)(text, (h0, c0)))\")" + "# h0 = torch.randn(6, len(nt_text), 512)\n", + "# c0 = torch.randn(6, len(nt_text), 512)\n", + "# print_eval(\"nt_text.nested_size(1)\")\n", + "# print_eval(\"nt_text.numel()\")\n", + "# print_eval(\"text.numel()\")\n", + "# print_eval(\"time(lambda: torch.nn.LSTM(256, 512, 6, batch_first=True)(nt_text, (h0, c0)))\")\n", + "# print_eval(\"time(lambda: torch.nn.LSTM(256, 512, 6, batch_first=True)(text, (h0, c0)))\")" ] }, { diff --git a/examples/naryops_and_reduce.ipynb b/examples/naryops_and_reduce.ipynb index 11bbb3d8..b22f46ec 100644 --- a/examples/naryops_and_reduce.ipynb +++ b/examples/naryops_and_reduce.ipynb @@ -6,7 +6,8 @@ "metadata": {}, "outputs": [], "source": [ - "from nestedtensor import torch\n", + "import torch\n", + "import nestedtensor\n", "from IPython.display import Markdown, display\n", "def print_eval(s):\n", " colorS = \"$ {}\".format(s)\n", @@ -43,7 +44,7 @@ "metadata": {}, "outputs": [], "source": [ - "nt = torch.nested_tensor(\n", + "nt = nestedtensor.nested_tensor(\n", "[\n", " [\n", " torch.tensor([1.0, 0.5, 1.5]),\n", @@ -55,7 +56,7 @@ " ]\n", "])\n", "\n", - "nt1 = torch.nested_tensor(\n", + "nt1 = nestedtensor.nested_tensor(\n", "[\n", " [\n", " torch.tensor([1.0, 0.5, 1.5]),\n", @@ -67,7 +68,7 @@ " ]\n", "])\n", "\n", - "nt2 = torch.nested_tensor(\n", + "nt2 = nestedtensor.nested_tensor(\n", "[\n", " [\n", " torch.tensor([1.0, 0.5, 1.5]),\n", diff --git a/examples/tensorwise.ipynb b/examples/tensorwise.ipynb index 2d8cc418..af204596 100644 --- a/examples/tensorwise.ipynb +++ b/examples/tensorwise.ipynb @@ -16,8 +16,9 @@ "metadata": {}, "outputs": [], "source": [ - "from nestedtensor import torch\n", - "from torch import tensorwise\n", + "import torch\n", + "import nestedtensor\n", + "from nestedtensor import tensorwise\n", "from IPython.display import Markdown, display\n", "\n", "def print_eval(s):\n", @@ -125,8 +126,8 @@ } ], "source": [ - "nt1 = torch.nested_tensor([a, b])\n", - "nt2 = torch.nested_tensor([b, a])\n", + "nt1 = nestedtensor.nested_tensor([a, b])\n", + "nt2 = nestedtensor.nested_tensor([b, a])\n", "print_eval('nt1')\n", "print_eval('nt2')" ] @@ -251,7 +252,7 @@ { "data": { "text/markdown": [ - "**$ sum_plus_one(a, torch.nested_tensor([nt1, nt2]))**" + "**$ sum_plus_one(a, nestedtensor.nested_tensor([nt1, nt2]))**" ], "text/plain": [ "" @@ -280,7 +281,7 @@ { "data": { "text/markdown": [ - "**$ sum_plus_one(nt1, torch.nested_tensor([nt1, nt2]))**" + "**$ sum_plus_one(nt1, nestedtensor.nested_tensor([nt1, nt2]))**" ], "text/plain": [ "" @@ -313,8 +314,8 @@ "print_eval('nt1')\n", "print_eval('nt2')\n", "print_eval('sum_plus_one(a, nt2)')\n", - "print_eval('sum_plus_one(a, torch.nested_tensor([nt1, nt2]))')\n", - "print_eval('sum_plus_one(nt1, torch.nested_tensor([nt1, nt2]))')" + "print_eval('sum_plus_one(a, nestedtensor.nested_tensor([nt1, nt2]))')\n", + "print_eval('sum_plus_one(nt1, nestedtensor.nested_tensor([nt1, nt2]))')" ] }, { @@ -563,20 +564,6 @@ "print_eval('nt2')\n", "print_eval('sum_plus_scalar(nt1, nt2, (2.0, 3.0))')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/text_classification.ipynb b/examples/text_classification.ipynb index d9bbfc6f..15fdad84 100644 --- a/examples/text_classification.ipynb +++ b/examples/text_classification.ipynb @@ -2,9 +2,21 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "Exception", + "evalue": "This is currently disabled!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0mURL\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"This is currently disabled!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mException\u001b[0m: This is currently disabled!" + ] + } + ], "source": [ "import re\n", "import requests\n", @@ -20,14 +32,17 @@ "from collections import Counter\n", "from collections import namedtuple\n", "\n", - "from nestedtensor import torch\n", + "import torch\n", + "import nestedtensor\n", + "\n", + "URL = \"https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz\"\n", "\n", - "URL = \"https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz\"" + "raise Exception(\"This example notebook is temporarily disabled!\")" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -74,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -86,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -109,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -122,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -158,28 +173,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total number of tokens: 27205880\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "time: 21s epoch: 0 lr: 1.000000 loss: 0.767902\n", - "time: 21s epoch: 1 lr: 0.810000 loss: 0.425534\n", - "time: 21s epoch: 2 lr: 0.729000 loss: 0.274846\n", - "time: 21s epoch: 3 lr: 0.656100 loss: 0.274525\n", - "time: 21s epoch: 4 lr: 0.590490 loss: 0.209720\n" - ] - } - ], + "outputs": [], "source": [ "num_tokens = sum(map(lambda x: len(x.text), data['train']))\n", "print(\"Total number of tokens: {}\".format(num_tokens))\n", @@ -212,17 +208,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test accuracy: 0.940569281578064\n" - ] - } - ], + "outputs": [], "source": [ "output = [(tb[0], model(tb[1].result().to('cuda')).argmax(1).cpu()) for tb in yield_data_futures(data['test'])]\n", "predictions = torch.cat(list(map(lambda x: x[1], output)))\n", @@ -244,18 +232,6 @@ "display_name": "Python 3", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.5" } }, "nbformat": 4, diff --git a/nestedtensor/__init__.py b/nestedtensor/__init__.py index 180e48ba..d6307355 100644 --- a/nestedtensor/__init__.py +++ b/nestedtensor/__init__.py @@ -1,10 +1,21 @@ import torch -if getattr(torch, "__IS_MONKEY_PATCHED_BY_NESTED_TENSOR", None) is None: - from .nested.monkey_patch import monkey_patch - torch = monkey_patch(torch) -# Confirm that this function was only applied once -assert torch.__IS_MONKEY_PATCHED_BY_NESTED_TENSOR == 1 from .nested.utils import tensorwise +from .nested.utils import is_nested_tensor + +from .nested.creation import as_nested_tensor +from .nested.creation import nested_tensor + +from .nested.masking import nested_tensor_from_tensor_mask +from .nested.masking import nested_tensor_from_padded_tensor + +from .nested.nested import NestedTensor + +from .nested.monkey_patch import monkey_patch + +from . import nested + from . import _C from ._C import _ListNestedTensor + +nested.monkey_patch.monkey_patch(NestedTensor) diff --git a/nestedtensor/csrc/nested_node.h b/nestedtensor/csrc/nested_node.h index d2d8e7a1..564a69ed 100644 --- a/nestedtensor/csrc/nested_node.h +++ b/nestedtensor/csrc/nested_node.h @@ -96,11 +96,11 @@ static inline _NestedNode _get_structure(PyObject* tensors) { Py_ssize_t i, n; n = PyObject_Length(tensors); PyObject* item; - if (n < 0) { + if (n < 0 || !PySequence_Check(tensors)) { throw python_error(); } for (i = 0; i < n; i++) { - item = PyList_GetItem(tensors, i); + item = PySequence_GetItem(tensors, i); _NestedNode node = _get_structure(item); meta_nodes.push_back(node); } diff --git a/nestedtensor/nested/__init__.py b/nestedtensor/nested/__init__.py index e69de29b..99558810 100644 --- a/nestedtensor/nested/__init__.py +++ b/nestedtensor/nested/__init__.py @@ -0,0 +1,2 @@ +from . import codegen +from . import utils diff --git a/nestedtensor/nested/bufferimpl.py b/nestedtensor/nested/bufferimpl.py index 7a77caea..b57f3002 100644 --- a/nestedtensor/nested/bufferimpl.py +++ b/nestedtensor/nested/bufferimpl.py @@ -3,6 +3,7 @@ import numbers from functools import wraps from . import utils +from . import nested import collections import os import nestedtensor @@ -126,9 +127,8 @@ def requires_grad_(self, requires_grad=True): return self def detach(self): - return nested.NestedTensor( - _BufferNestedTensor(self.get_buffer().detach, - self.nested_size(), self.nested_stride())) + return _BufferNestedTensor(self.get_buffer().detach, + self.nested_size(), self.nested_stride()) def backward(self, gradient, retain_graph, create_graph): for t, g in zip(self.unbind(), gradient.unbind()): @@ -207,8 +207,8 @@ def _size(nested_size): return _size(self.nested_size()) def to(self, *args, **kwargs): - return torch.NestedTensor(_BufferNestedTensor(self.get_buffer().to(*args, **kwargs), - self.nested_size(), self.nested_stride())) + return _BufferNestedTensor(self.get_buffer().to(*args, **kwargs), + self.nested_size(), self.nested_stride()) def numel(self): return self.get_buffer().numel() diff --git a/nestedtensor/nested/monkey_patch.py b/nestedtensor/nested/monkey_patch.py index 012ef9da..d99dfcf9 100644 --- a/nestedtensor/nested/monkey_patch.py +++ b/nestedtensor/nested/monkey_patch.py @@ -1,35 +1,4 @@ -def _check_meaningful_overwrite(cls, method_name): - import os - DEBUG = int(os.getenv("DEBUG", 0)) - - class DefaultClass(object): - pass - - if DEBUG and getattr(cls, method_name, False) and not getattr(DefaultClass, method_name, False): - raise Exception("WARNING: " + method_name + " already exists " - "and not part of default class") - - -def set_module(module, name, wrapper): - new_fn = wrapper(getattr(module, name)) - new_fn.__MODULE_FUNCTION_SUPPORTED_BY_NESTED_TENSOR = 1 - setattr(module, name, new_fn) - return module - - -def set_nt_method(name, wrapper): - import torch - from .nested import NestedTensor - _check_meaningful_overwrite(NestedTensor, name) - setattr(NestedTensor, name, wrapper(getattr(torch.Tensor, name))) - - -def set_module_and_nt_method(module, name, wrapper): - set_nt_method(name, wrapper) - return set_module(module, name, wrapper) - - -def monkey_patch(module): +def monkey_patch(NestedTensor): """ Functions that are being skipped are sometimes not skipped for a good reason other than a lack of completed implemetation @@ -38,28 +7,44 @@ def monkey_patch(module): import os DEBUG = int(os.getenv("DEBUG", 0)) - from . import utils - from .nested import NestedTensor - from . import creation - from . import masking - from . import codegen - from . import functions + + from nestedtensor.nested import codegen + from nestedtensor.nested import functions import torch + from nestedtensor.nested import utils - module.__IS_MONKEY_PATCHED_BY_NESTED_TENSOR = getattr( - module, "__IS_MONKEY_PATCHED_BY_NESTED_TENSOR", 0) + 1 - module.tensorwise = utils.tensorwise - module.is_nested_tensor = utils.is_nested_tensor + function_dispatch = {} - # > PyTorch constructors - module.as_nested_tensor = creation.as_nested_tensor - module.nested_tensor = creation.nested_tensor - module.nested_tensor_from_tensor_mask = masking.nested_tensor_from_tensor_mask - module.nested_tensor_from_padded_tensor = masking.nested_tensor_from_padded_tensor - # < + def _check_meaningful_overwrite(cls, method_name): + import os + DEBUG = int(os.getenv("DEBUG", 0)) + + class DefaultClass(object): + pass + + if DEBUG and getattr(cls, method_name, False) and not getattr(DefaultClass, method_name, False): + raise Exception("WARNING: " + method_name + " already exists " + "and not part of default class") + + def set_nt_method(name, wrapper): + _check_meaningful_overwrite(NestedTensor, name) + setattr(NestedTensor, name, wrapper(getattr(torch.Tensor, name))) + + def set_wrapped_torch_function(function_name, wrapper): + function_dispatch[getattr(torch, function_name)] = wrapper( + getattr(torch, function_name)) + + def set_function(key, function): + function_dispatch[key] = function # > Python data model methods. We skip functions that torch.Tensor doesn't define. # --- Python binary arithmetic operations + + for function_name in codegen.get_python_binary_arithmetic_operations(): + if function_name in ['truediv', 'floordiv', 'mod', 'divmod', 'lshift', 'rshift', 'and', 'xor', 'or']: + continue + set_wrapped_torch_function(function_name, utils.pointwise()) + for function_name in codegen.get_python_binary_arithmetic_operations(): if function_name in ['divmod']: continue @@ -76,6 +61,11 @@ def monkey_patch(module): set_nt_method("__r" + function_name + '__', utils.pointwise()) # --- Python unary arithmetic operations + for function_name in ['neg', 'pos', 'abs', 'invert']: + if function_name in ['pos', 'invert']: + continue + set_wrapped_torch_function(function_name, utils.pointwise()) + for function_name in ['neg', 'pos', 'abs', 'invert']: if function_name in ['pos']: continue @@ -89,29 +79,23 @@ def monkey_patch(module): # > PyTorch pointwise operations # --- Module and Tensor pointwise functions and methods for function_name in codegen.get_pointwise_functions(): + set_nt_method(function_name + '_', utils.pointwise()) if function_name in ['fill']: continue + set_wrapped_torch_function(function_name, utils.pointwise()) set_nt_method(function_name, utils.pointwise()) - - for function_name in codegen.get_pointwise_functions(): - if function_name in ['fill']: - continue - module = set_module(module, function_name, utils.pointwise()) - - # --- Tensor pointwise in-place methods - for function_name in codegen.get_pointwise_functions(): - set_nt_method(function_name + '_', utils.pointwise()) # < # > PyTorch reduction operations # --- Module and Tensor reductions for function_name in codegen.get_complete_reductions(): - module = set_module_and_nt_method( - module, function_name, utils.reduction()) + set_wrapped_torch_function(function_name, utils.reduction()) + set_nt_method(function_name, utils.reduction()) for function_name in codegen.get_tensorwise_reductions(): - module = set_module_and_nt_method( - module, function_name, utils.reduction(support_nested_dim=False)) + set_wrapped_torch_function( + function_name, utils.reduction(support_nested_dim=False)) + set_nt_method(function_name, utils.reduction(support_nested_dim=False)) # < # > PyTorch conversion methods @@ -121,6 +105,7 @@ def monkey_patch(module): # > PyTorch BLAS and LAPACK operations for function_name in codegen.get_blas_lapack_ops(): + set_wrapped_torch_function(function_name, utils.tensorwise()) if function_name in ['chain_matmul', 'lu_unpack', 'matrix_rank', 'trapz']: continue set_nt_method(function_name, utils.tensorwise()) @@ -135,13 +120,9 @@ def monkey_patch(module): continue set_nt_method(function_name + '_', utils.tensorwise()) - for function_name in codegen.get_blas_lapack_ops(): - module = set_module( - module, function_name, utils.tensorwise()) - # < - # > PyTorch BLAS and LAPACK operations for function_name in codegen.get_other_ops(): + set_wrapped_torch_function(function_name, utils.tensorwise()) # Custom implementation if function_name in ['flatten']: continue @@ -158,21 +139,14 @@ def monkey_patch(module): continue set_nt_method(function_name + '_', utils.tensorwise()) - for function_name in codegen.get_other_ops(): - if function_name in ['flatten']: - module = set_module( - module, function_name, utils.tensorwise(dim_args=[1, 2, 'start_dim', 'end_dim'])) - else: - module = set_module( - module, function_name, utils.tensorwise()) # < - # # > PyTorch random sampling operations + # > PyTorch random sampling operations for function_name in codegen.get_random_sampling_operations(): - if function_name in ['cauchy', 'exponential', 'geometric', 'log_normal', 'uniform']: + if function_name in ['cauchy', 'exponential', 'geometric', 'log_normal', + 'normal', 'random', 'uniform']: continue - module = set_module( - module, function_name, utils.tensorwise()) + set_wrapped_torch_function(function_name, utils.tensorwise()) for function_name in codegen.get_random_sampling_operations(): if function_name in ['cauchy', 'exponential', 'geometric', 'log_normal', @@ -189,11 +163,10 @@ def monkey_patch(module): # TODO: low-pri: improved error reporting for signal_dim # > PyTorch spectral operations for function_name in codegen.get_fft_ops(): - module = set_module_and_nt_method( - module, function_name, utils.tensorwise(dim_args=[1, 'signal_dim'])) + set_nt_method(function_name, utils.tensorwise( + dim_args=[1, 'signal_dim'])) for function_name in codegen.get_stft_ops(): - module = set_module_and_nt_method( - module, function_name, utils.tensorwise()) + set_nt_method(function_name, utils.tensorwise()) # < # --- AD HOC @@ -204,26 +177,26 @@ def monkey_patch(module): for function_name in ['clone', 'detach']: set_nt_method(function_name, utils.tensorwise()) - # By default everything is tensorwise, but for improved semantics - # we extend the e.g. conv2d that's broadcast to also accept images - # without a leading batch. - # TODO: Need to split out dim functions - # TODO: Rerun pipelines - for function_name in codegen.get_functionals(): - # NOTE: They have Custom implementations - if function_name in ['conv2d', 'embedding_bag', 'linear', - 'batch_norm', 'max_pool2d', 'interpolate']: - continue - if function_name in ['relu', 'relu_']: - set_module(module.nn.functional, function_name, utils.pointwise()) - else: - set_module(module.nn.functional, function_name, utils.tensorwise()) + # # By default everything is tensorwise, but for improved semantics + # # we extend the e.g. conv2d that's broadcast to also accept images + # # without a leading batch. + # # TODO: Need to split out dim functions + # # TODO: Rerun pipelines + # for function_name in codegen.get_functionals(): + # # NOTE: They have Custom implementations + # if function_name in ['conv2d', 'embedding_bag', 'linear', + # 'batch_norm', 'max_pool2d', 'interpolate']: + # continue + # if function_name in ['relu', 'relu_']: + # set_module(module.nn.functional, function_name, utils.pointwise()) + # else: + # set_module(module.nn.functional, function_name, utils.tensorwise()) set_nt_method('log_softmax', utils.tensorwise(dim_args=[1, 'dim'])) - # TODO: Might need dispatch wrapper? - module.mv = utils.tensorwise()(torch.mv) - module.mm = utils.tensorwise()(torch.mm) + # # TODO: Might need dispatch wrapper? + # functions['mv'] = utils.tensorwise()(torch.mv) + # functions['mm'] = utils.tensorwise()(torch.mm) # --- custom functions # set_nt_method('mm', functions.mm) @@ -235,17 +208,17 @@ def monkey_patch(module): # setattr(module, function_name, getattr(functions, function_name)) # set_nt_method(function_name, getattr(functions, function_name)) - module.nn.functional.conv2d = functions.conv2d - module.nn.functional.max_pool2d = functions.max_pool2d - module.nn.functional.embedding_bag = functions.embedding_bag - module.nn.functional.linear = functions.linear - module.nn.functional.batch_norm = functions.batch_norm - module.nn.functional.interpolate = functions.interpolate - # module.nn.functional.nll_loss = functions.nll_loss + set_function(torch.conv2d, functions.conv2d) + set_function(torch.max_pool2d, functions.max_pool2d) + set_function(torch.embedding_bag, functions.embedding_bag) + set_function(torch.batch_norm, functions.batch_norm) + # module.nn.functional.linear = functions.linear + # module.nn.functional.interpolate = functions.interpolate + # # module.nn.functional.nll_loss = functions.nll_loss - # --- custom modules - module.nn.modules.rnn.LSTM.forward = functions.lstm_forward + # # --- custom modules + # module.nn.modules.rnn.LSTM.forward = functions.lstm_forward - module.NestedTensor = NestedTensor + # module.NestedTensor = NestedTensor - return module + setattr(NestedTensor, '_NestedTensor__function_dispatch', function_dispatch) diff --git a/nestedtensor/nested/nested.py b/nestedtensor/nested/nested.py index 9e29f358..48a35c0c 100644 --- a/nestedtensor/nested/nested.py +++ b/nestedtensor/nested/nested.py @@ -2,6 +2,7 @@ import numbers from functools import wraps from . import masking +from . import monkey_patch import collections import os @@ -190,7 +191,7 @@ def compare_sizes(size, other_size): def to(self, *args, **kwargs): # to is never in-place, but it has autograd support (for float and double) via copy - return self._impl.to(*args, **kwargs) + return NestedTensor(self._impl.to(*args, **kwargs)) def numel(self): return self._impl.numel() @@ -220,7 +221,7 @@ def unbind(self, dim=0): return tuple(t if torch.is_tensor(t) else NestedTensor(t) for t in self._impl.unbind()) else: unbound = tuple(t.unbind(dim - 1) for t in self.unbind(dim - 1)) - return tuple(torch.nested_tensor(t) for t in zip(*unbound)) + return tuple(creation.nested_tensor(t) for t in zip(*unbound)) def to_tensor(self, dim=0): """ @@ -237,7 +238,7 @@ def to_tensor(self, dim=0): if self.nested_dim() == 1: return self unbound = [t.to_tensor(dim=dim - 1) for t in self.unbind()] - return torch.nested_tensor(unbound) + return creation.nested_tensor(unbound) def __repr__(self): # TODO: This relies on the fact that repr is not implemented compliant with @@ -284,6 +285,13 @@ def nested_stride(self, dim=None): # --- dependent on impl ends --- + def __torch_function__(self, func, args=(), kwargs=None): + _local_func = None + if func in NestedTensor.__function_dispatch: + _local_func = NestedTensor.__function_dispatch[func] + return _local_func(*args) if kwargs is None else _local_func(*args, **kwargs) + raise NotImplementedError("NestedTensor doesn't support function {}".format(func)) + def __bool__(self): raise NotImplementedError( "This has not been covered by NestedTensor 0.0.1") @@ -297,7 +305,7 @@ def __getitem__(self, key): if isinstance(key, numbers.Number): return self.unbind()[key] if isinstance(key, slice): - return torch.as_nested_tensor(self.unbind()[key]) + return creation.as_nested_tensor(self.unbind()[key]) assert isinstance(key, tuple) if key[0] == Ellipsis: raise NotImplementedError( @@ -306,7 +314,7 @@ def __getitem__(self, key): selected_tensors = self.unbind()[key[0]] if len(key) == 1: return selected_tensors - return torch.as_nested_tensor([t[key[1:]] for t in selected_tensors]) + return creation.as_nested_tensor([t[key[1:]] for t in selected_tensors]) def __iter__(self): return iter(self.unbind()) @@ -320,9 +328,9 @@ def to_nested_tensor(self, dim=0): raise ValueError("Given dimension is already nested") else: if self.nested_dim == dim: - return torch.nested_tensor(list(t.unbind() for t in self.unbind())) + return creation.nested_tensor(list(t.unbind() for t in self.unbind())) else: - return torch.nested_tensor(list(t.to_nested_tensor(dim - 1) for t in self.unbind())) + return creation.nested_tensor(list(t.to_nested_tensor(dim - 1) for t in self.unbind())) def to_list(self): if self.nested_dim() == 1: diff --git a/nestedtensor/version.py b/nestedtensor/version.py index b66624c4..f482cda6 100644 --- a/nestedtensor/version.py +++ b/nestedtensor/version.py @@ -1,5 +1,5 @@ -__version__ = '0.0.1.dev2019121019+f2c3cb6' -git_version = 'f2c3cb64575e8601fe6dee58f84373dabb65ead3' +__version__ = '0.0.1.dev2019121121+a4d5d6c' +git_version = 'a4d5d6cfbe1fcf4fdb8f871d24f89e86ffb22aab' from nestedtensor import _C if hasattr(_C, 'CUDA_VERSION'): cuda = _C.CUDA_VERSION diff --git a/test/test_nested_tensor_autograd.py b/test/test_nested_tensor_autograd.py index 8381c463..a55069e0 100644 --- a/test/test_nested_tensor_autograd.py +++ b/test/test_nested_tensor_autograd.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase diff --git a/test/test_nested_tensor_class.py b/test/test_nested_tensor_class.py index b9d2dbe5..f9bb801e 100644 --- a/test/test_nested_tensor_class.py +++ b/test/test_nested_tensor_class.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase @@ -17,30 +17,30 @@ def test_nested_constructor(self): # TODO: Shouldn't be constructable nested_tensors = [utils.gen_nested_tensor(i, i, 3) for i in range(1, num_nested_tensor)] - nested_tensor = torch.nested_tensor(nested_tensors) + nested_tensor = nestedtensor.nested_tensor(nested_tensors) def test_constructor(self): tensors = [] num_tensors = 16 for i in range(num_tensors): tensors.append(utils.gen_float_tensor(i, (i + 1, 128, 128))) - nested_tensor = torch.nested_tensor(tensors) + nested_tensor = nestedtensor.nested_tensor(tensors) for i in range(num_tensors): tensors[i].mul_(i + 2) for i in range(num_tensors): self.assertTrue((tensors[i] != nested_tensor.unbind()[i]).all()) self.assertRaises( - ValueError, lambda: torch.nested_tensor(torch.tensor([3.0]))) - self.assertRaises(ValueError, lambda: torch.nested_tensor( - torch.nested_tensor([torch.tensor([3.0])]))) - self.assertRaises(ValueError, lambda: torch.nested_tensor( - [torch.tensor([2.0]), torch.nested_tensor([torch.tensor([3.0])])])) - self.assertRaises(ValueError, lambda: torch.nested_tensor(4.0)) + ValueError, lambda: nestedtensor.nested_tensor(torch.tensor([3.0]))) + self.assertRaises(ValueError, lambda: nestedtensor.nested_tensor( + nestedtensor.nested_tensor([torch.tensor([3.0])]))) + self.assertRaises(ValueError, lambda: nestedtensor.nested_tensor( + [torch.tensor([2.0]), nestedtensor.nested_tensor([torch.tensor([3.0])])])) + self.assertRaises(ValueError, lambda: nestedtensor.nested_tensor(4.0)) def test_default_constructor(self): - self.assertRaises(TypeError, lambda: torch.nested_tensor()) + self.assertRaises(TypeError, lambda: nestedtensor.nested_tensor()) # nested_dim is 1 and dim is 1 too. - default_nested_tensor = torch.nested_tensor([]) + default_nested_tensor = nestedtensor.nested_tensor([]) default_tensor = torch.tensor([]) self.assertEqual(default_nested_tensor.nested_dim(), 1) self.assertEqual(default_nested_tensor.nested_size(), ()) @@ -54,40 +54,40 @@ def test_default_constructor(self): default_tensor.is_pinned()) def test_scalar_constructor(self): - dim_one_nested_tensor = torch.nested_tensor([1.0]) + dim_one_nested_tensor = nestedtensor.nested_tensor([1.0]) self.assertEqual(dim_one_nested_tensor.dim(), 1) self.assertEqual(dim_one_nested_tensor.nested_dim(), 1) def test_nested_size(self): - a = torch.nested_tensor( + a = nestedtensor.nested_tensor( [torch.rand(1, 2), torch.rand(2, 3), torch.rand(4, 5)]) na = (torch.Size([1, 2]), torch.Size([2, 3]), torch.Size([4, 5])) self.assertEqual(a.nested_size(), na) values = [torch.rand(1, 2) for i in range(10)] values = [values[1:i] for i in range(2, 10)] - nt = torch.nested_tensor(values) + nt = nestedtensor.nested_tensor(values) nts = nt.nested_size(1) lens = tuple(map(len, values)) self.assertTrue(nts == lens) def test_len(self): - a = torch.nested_tensor([torch.tensor([1, 2]), + a = nestedtensor.nested_tensor([torch.tensor([1, 2]), torch.tensor([3, 4]), torch.tensor([5, 6]), torch.tensor([7, 8])]) self.assertEqual(len(a), 4) - a = torch.nested_tensor([torch.tensor([1, 2]), + a = nestedtensor.nested_tensor([torch.tensor([1, 2]), torch.tensor([7, 8])]) self.assertEqual(len(a), 2) - a = torch.nested_tensor([torch.tensor([1, 2])]) + a = nestedtensor.nested_tensor([torch.tensor([1, 2])]) self.assertEqual(len(a), 1) def test_equal(self): - a1 = torch.nested_tensor([torch.tensor([1, 2]), + a1 = nestedtensor.nested_tensor([torch.tensor([1, 2]), torch.tensor([7, 8])]) - a2 = torch.nested_tensor([torch.tensor([1, 2]), + a2 = nestedtensor.nested_tensor([torch.tensor([1, 2]), torch.tensor([7, 8])]) - a3 = torch.nested_tensor([torch.tensor([3, 4]), + a3 = nestedtensor.nested_tensor([torch.tensor([3, 4]), torch.tensor([5, 6])]) # Just exercising them until we have __bool__, all() etc. self.assertTrue((a1 == a2).all()) @@ -96,7 +96,7 @@ def test_equal(self): self.assertTrue(not (a1 == a3).any()) def test_nested_dim(self): - nt = torch.nested_tensor([torch.tensor(3)]) + nt = nestedtensor.nested_tensor([torch.tensor(3)]) self.assertTrue(nt.nested_dim() == 1) for i in range(2, 5): nt = utils.gen_nested_tensor(i, i, 3) @@ -105,22 +105,22 @@ def test_nested_dim(self): def test_unbind(self): a = torch.tensor([1, 2]) b = torch.tensor([7, 8]) - nt = torch.nested_tensor([a, b]) + nt = nestedtensor.nested_tensor([a, b]) a1, b1 = nt.unbind() self.assertTrue((a == a1).all()) self.assertTrue((b == b1).all()) a = utils.gen_float_tensor(1, (2, 3)).add_(1) - nt = torch.nested_tensor([a]) + nt = nestedtensor.nested_tensor([a]) self.assertTrue((a == nt.unbind()[0]).all()) def test_size(self): - a1 = torch.nested_tensor([[torch.rand(1, 8), + a1 = nestedtensor.nested_tensor([[torch.rand(1, 8), torch.rand(3, 8)], [torch.rand(7, 8)]]) - a2 = torch.nested_tensor([torch.rand(1, 2), + a2 = nestedtensor.nested_tensor([torch.rand(1, 2), torch.rand(1, 8)]) - a3 = torch.nested_tensor([torch.rand(3, 4), + a3 = nestedtensor.nested_tensor([torch.rand(3, 4), torch.rand(5, 4)]) self.assertTrue(a1.size() == (2, None, None, 8)) self.assertTrue(a2.size() == (2, 1, None)) @@ -131,7 +131,7 @@ def test_contiguous(self): for i in range(1, 10): # data = gen_nested_list(1, 2, 3, size_low=1, size_high=3) data = [[torch.rand(1, 2), torch.rand(3, 4)], [torch.rand(5, 6)]] - nt = torch.nested_tensor(data) + nt = nestedtensor.nested_tensor(data) self.assertTrue(nt.is_contiguous()) # buf = nt.flatten() self.assertTrue((nt == nt).all()) diff --git a/test/test_nested_tensor_functional.py b/test/test_nested_tensor_functional.py index b99d25ab..dc06b39e 100644 --- a/test/test_nested_tensor_functional.py +++ b/test/test_nested_tensor_functional.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase @@ -17,7 +17,7 @@ def test_nll_loss(self): def test_addmm(self): a, b = torch.rand(5), torch.rand(4, 5) - nt = torch.nested_tensor( + nt = nestedtensor.nested_tensor( [torch.rand(1, 4), torch.rand(1, 4), torch.rand(4, 4)]) def test_conv2d(self): @@ -31,7 +31,7 @@ def _conv2d(input, *args, **kwargs): weight = torch.rand(64, 3, 7, 7) tf = nestedtensor.tensorwise()(_conv2d) images = [torch.rand(3, (i * 16) % 40 + 40, (i * 16) % 50 + 40) for i in range(128)] - nt = torch.nested_tensor(images) + nt = nestedtensor.nested_tensor(images) result = tf(nt, weight) result2 = torch.nn.functional.conv2d(nt, weight) for r, r2 in zip(result, result2): diff --git a/test/test_nested_tensor_masking.py b/test/test_nested_tensor_masking.py index 5d2df63c..190051ec 100644 --- a/test/test_nested_tensor_masking.py +++ b/test/test_nested_tensor_masking.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase @@ -16,7 +16,7 @@ class TestTensorMask(TestCase): def test_simple(self): values = [torch.rand(i) for i in range(10)] random.shuffle(values) - nt = torch.nested_tensor(values) + nt = nestedtensor.nested_tensor(values) tensor, mask = nt.to_tensor_mask() def test_nested(self): @@ -26,17 +26,17 @@ def test_nested(self): def test_tensor_mask(self): nt = utils.gen_nested_tensor(2, 2, 2, size_low=1, size_high=2) tensor, mask = nt.to_tensor_mask() - nt1 = torch.nested_tensor_from_tensor_mask( + nt1 = nestedtensor.nested_tensor_from_tensor_mask( tensor, mask, nested_dim=nt.nested_dim()) self.assertTrue((nt == nt1).all()) - nt2 = torch.nested_tensor_from_tensor_mask( + nt2 = nestedtensor.nested_tensor_from_tensor_mask( tensor, mask) self.assertTrue((nt == nt2).all()) def test_tensor_mask_lowdim(self): values = [torch.rand(1, 2) for i in range(10)] values = [values[1:i] for i in range(2, 10)] - nt = torch.nested_tensor(values) + nt = nestedtensor.nested_tensor(values) tensor, mask = nt.to_tensor_mask() self.assertTrue([len(value) for value in values] == list(mask.sum(-1))) diff --git a/test/test_nested_tensor_nary.py b/test/test_nested_tensor_nary.py index b3ec5c00..0d6644f5 100644 --- a/test/test_nested_tensor_nary.py +++ b/test/test_nested_tensor_nary.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase @@ -26,11 +26,11 @@ def _test_unary(self): if func__ in ['mvlgamma']: data = utils.nested_map(lambda x: x.clamp(min=1), data) - a1 = torch.nested_tensor(data) - a3 = torch.nested_tensor(data) + a1 = nestedtensor.nested_tensor(data) + a3 = nestedtensor.nested_tensor(data) func_ = getattr(torch, func__) - method_ = getattr(torch.NestedTensor, func__) - method_inplace_ = getattr(torch.NestedTensor, func__ + "_") + method_ = getattr(nestedtensor.NestedTensor, func__) + method_inplace_ = getattr(nestedtensor.NestedTensor, func__ + "_") if func__ in ['clamp']: def func(x, out=None): @@ -85,7 +85,7 @@ def method_inplace(x): return method_inplace_(x, 0.3) method = method_ method_inplace = method_inplace_ - a2 = torch.nested_tensor(utils.nested_map(func, data)) + a2 = nestedtensor.nested_tensor(utils.nested_map(func, data)) self.assertTrue(a1.nested_dim() == a2.nested_dim()) self.assertTrue(a2.nested_dim() == a3.nested_dim()) @@ -110,11 +110,11 @@ def _test_binary(self): b = utils.gen_float_tensor(2, (2, 3)) c = utils.gen_float_tensor(3, (2, 3)) # The constructor is supposed to copy! - a1 = torch.nested_tensor([a, b]) - a2 = torch.nested_tensor([b, c]) - a1_l = torch.as_nested_tensor([a.clone(), b.clone()]) - a2_l = torch.as_nested_tensor([b.clone(), c.clone()]) - a3 = torch.nested_tensor([getattr(torch, func)(a, b), + a1 = nestedtensor.nested_tensor([a, b]) + a2 = nestedtensor.nested_tensor([b, c]) + a1_l = nestedtensor.as_nested_tensor([a.clone(), b.clone()]) + a2_l = nestedtensor.as_nested_tensor([b.clone(), c.clone()]) + a3 = nestedtensor.nested_tensor([getattr(torch, func)(a, b), getattr(torch, func)(b, c)]) self.assertTrue((a3 == getattr(torch, func)(a1_l, a2_l)).all()) self.assertTrue((a3 == getattr(torch, func)(a1, a2)).all()) diff --git a/test/test_nested_tensor_tensorwise.py b/test/test_nested_tensor_tensorwise.py index a1a05154..79a25123 100644 --- a/test/test_nested_tensor_tensorwise.py +++ b/test/test_nested_tensor_tensorwise.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase @@ -20,8 +20,8 @@ def simple_fn(t1, t2): a = torch.tensor([1, 2]) b = torch.tensor([7, 8]) - nt1 = torch.nested_tensor([a, b]) - nt2 = torch.nested_tensor([b, a]) + nt1 = nestedtensor.nested_tensor([a, b]) + nt2 = nestedtensor.nested_tensor([b, a]) def test_tensorwise_nested_dim_2(self): @@ -31,8 +31,8 @@ def simple_fn(t1, t2): a = torch.tensor([1, 2]) b = torch.tensor([7, 8]) - nt1 = torch.nested_tensor([[a, b], [b]]) - nt2 = torch.nested_tensor([[b, a], [a]]) + nt1 = nestedtensor.nested_tensor([[a, b], [b]]) + nt2 = nestedtensor.nested_tensor([[b, a], [a]]) nt3 = simple_fn(nt1, nt2) def test_tensorwise_scalar(self): @@ -43,8 +43,8 @@ def simple_fn_scalar(t1, t2, scalar): a = torch.tensor([1, 2]) b = torch.tensor([7, 8]) - nt1 = torch.nested_tensor([a, b]) - nt2 = torch.nested_tensor([b, a]) + nt1 = nestedtensor.nested_tensor([a, b]) + nt2 = nestedtensor.nested_tensor([b, a]) c = simple_fn_scalar(a, b, 2.0) nt3 = simple_fn_scalar(nt1, nt2, (2.0, 3.0)) self.assertTrue((c == nt3[0]).all()) @@ -60,8 +60,8 @@ def simple_fn(t1, t2, t3=None): a = torch.tensor([1, 2]) b = torch.tensor([7, 8]) - nt1 = torch.nested_tensor([a, b]) - nt2 = torch.nested_tensor([b, a]) + nt1 = nestedtensor.nested_tensor([a, b]) + nt2 = nestedtensor.nested_tensor([b, a]) c1 = simple_fn(a, b, t3=torch.tensor((0.5, 0.7))) c2 = simple_fn(b, a, t3=torch.tensor((0.5, 0.7))) nt3 = simple_fn(nt1, nt2, t3=torch.tensor((0.5, 0.7))) diff --git a/test/utils.py b/test/utils.py index 05eb087e..f73916cc 100644 --- a/test/utils.py +++ b/test/utils.py @@ -2,7 +2,7 @@ import functools import pdb import sys -from nestedtensor import torch +import torch import nestedtensor import unittest from unittest import TestCase @@ -105,4 +105,4 @@ def nested_map(fn, data): def gen_nested_tensor(seed, nested_dim, tensor_dim, size_low=1, size_high=10): - return torch.nested_tensor(gen_nested_list(seed, nested_dim, tensor_dim, size_low=size_low, size_high=size_high)) + return nestedtensor.nested_tensor(gen_nested_list(seed, nested_dim, tensor_dim, size_low=size_low, size_high=size_high))