*This model was published in HF papers on 2026-04-13 and contributed to Hugging Face Transformers on 2026-07-06.*
# TIPSv2 DPT
## Overview
[TIPSv2](./tipsv2) with dense prediction transformer head (DPT), see the [TIPSv2](./tipsv2) documentation for more information.
This model was contributed by [guarin](https://huggingface.co/guarin).
The original code can be found [here](https://github.com/google-deepmind/tips).
You can find all the original TIPSv2 DPT checkpoints under the [TIPSv2](https://huggingface.co/collections/google/tipsv2) collection.
> [!TIP]
> See [TIPSv2](./tipsv2) for the TIPSv2 vision and language backbones as well as zero shot image classification.
```python
from transformers import pipeline
pipe = pipeline(task="depth-estimation", model="google/tipsv2-b14-dpt", device_map="auto")
out = pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
out["depth"] # PIL Image normalized to [min, max] of predicted_depth
# Visualization
import matplotlib.pyplot as plt
from transformers.image_utils import load_image
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
figure, axes = plt.subplots(1, 2, figsize=(10, 5))
axes[0].imshow(image)
axes[1].imshow(out["depth"])
axes[1].set_title("Depth")
for axis in axes:
axis.axis("off")
plt.show()
```
```python
from transformers import pipeline
pipe = pipeline(task="image-segmentation", model="google/tipsv2-b14-dpt", device_map="auto")
out = pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
out[0]["label"] # string class label
out[0]["mask"] # PIL Image with binary mask set to class id
# Visualization
import matplotlib.pyplot as plt
from transformers.image_utils import load_image
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
figure, axes = plt.subplots(1, 2, figsize=(10, 5))
axes[0].imshow(image)
axes[1].imshow(out[0]["mask"], cmap="gray")
axes[1].set_title(out[0]["label"])
for axis in axes:
axis.axis("off")
plt.show()
```
Use [`Tipsv2DptForDensePrediction`] to run all three tasks (depth, normals, and segmentation) in a single forward pass over a shared backbone.
```python
import torch
from transformers import Tipsv2DptForDensePrediction, AutoImageProcessor
from transformers.image_utils import load_image
model_id = "google/tipsv2-b14-dpt"
model = Tipsv2DptForDensePrediction.from_pretrained(model_id, device_map="auto")
image_processor = AutoImageProcessor.from_pretrained(model_id)
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
inputs = image_processor(images=image, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model(**inputs)
outputs.predicted_depth # (batch_size, height, width) tensor with predicted depth in meters
outputs.normals # (batch_size, 3, height, width) tensor with normals in XYZ format (unnormalized)
outputs.segmentation_logits # # (batch_size, config.num_labels, height, width) tensor with segmentation logits
depth_results = image_processor.post_process_depth_estimation(outputs, target_sizes=[(image.height, image.width)])
normal_results = image_processor.post_process_normal_estimation(outputs, target_sizes=[(image.height, image.width)])
segmentation_results = image_processor.post_process_semantic_segmentation(outputs, target_sizes=[(image.height, image.width)])
predicted_depth = depth_results[0]["predicted_depth"] # (height, width) tensor with predicted depth in meters
normals = normal_results[0]["normals"] # (3, height, width) tensor with normals in XYZ format (L2-normalized)
segmentation = segmentation_results[0] # (height, width) tensor with class ids
# Visualization
import matplotlib.pyplot as plt
# Convert L2-normalized normals in [-1, 1] to RGB in [0, 255]
normals_rgb = ((normals + 1.0) / 2.0 * 255.0).clamp(0, 255).to(torch.uint8)
figure, axes = plt.subplots(2, 2, figsize=(10, 6))
axes[0, 0].imshow(image)
axes[0, 1].imshow(predicted_depth.cpu())
axes[0, 1].set_title("Depth")
axes[1, 0].imshow(normals_rgb.permute(1, 2, 0).cpu())
axes[1, 0].set_title("Normals")
axes[1, 1].imshow(segmentation.cpu())
axes[1, 1].set_title("Segmentation")
for axis in axes.flat:
axis.axis("off")
plt.show()
```
```python
import torch
from transformers import AutoModelForDepthEstimation, AutoImageProcessor
from transformers.image_utils import load_image
model_id = "google/tipsv2-b14-dpt"
model = AutoModelForDepthEstimation.from_pretrained(model_id, device_map="auto")
image_processor = AutoImageProcessor.from_pretrained(model_id)
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
inputs = image_processor(images=image, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model(**inputs)
results = image_processor.post_process_depth_estimation(outputs, target_sizes=[(image.height, image.width)])
predicted_depth = results[0]["predicted_depth"] # (height, width) tensor with predicted depth in meters
# Visualization
import matplotlib.pyplot as plt
figure, axes = plt.subplots(1, 2, figsize=(10, 5))
axes[0].imshow(image)
axes[1].imshow(predicted_depth.cpu())
axes[1].set_title("Depth")
for axis in axes:
axis.axis("off")
plt.show()
```
```python
import torch
from transformers import Tipsv2DptForNormalEstimation, AutoImageProcessor
from transformers.image_utils import load_image
model_id = "google/tipsv2-b14-dpt"
model = Tipsv2DptForNormalEstimation.from_pretrained(model_id, device_map="auto")
image_processor = AutoImageProcessor.from_pretrained(model_id)
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
inputs = image_processor(images=image, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model(**inputs)
results = image_processor.post_process_normal_estimation(outputs, target_sizes=[(image.height, image.width)])
normals = results[0]["normals"] # (3, height, width) tensor with normals in XYZ format (L2-normalized)
# Visualization
import matplotlib.pyplot as plt
# Convert L2-normalized normals in [-1, 1] to RGB in [0, 255]
normals_rgb = ((normals + 1.0) / 2.0 * 255.0).clamp(0, 255).to(torch.uint8)
figure, axes = plt.subplots(1, 2, figsize=(10, 5))
axes[0].imshow(image)
axes[1].imshow(normals_rgb.permute(1, 2, 0).cpu())
axes[1].set_title("Normals")
for axis in axes:
axis.axis("off")
plt.show()
```
```python
import torch
from transformers import AutoModelForSemanticSegmentation, AutoImageProcessor
from transformers.image_utils import load_image
model_id = "google/tipsv2-b14-dpt"
model = AutoModelForSemanticSegmentation.from_pretrained(model_id, device_map="auto")
image_processor = AutoImageProcessor.from_pretrained(model_id)
image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/room.jpg")
inputs = image_processor(images=image, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model(**inputs)
results = image_processor.post_process_semantic_segmentation(outputs, target_sizes=[(image.height, image.width)])
segmentation_map = results[0] # (height, width) tensor with class ids
# Visualization
import matplotlib.pyplot as plt
figure, axes = plt.subplots(1, 2, figsize=(10, 5))
axes[0].imshow(image)
axes[1].imshow(segmentation_map.cpu())
axes[1].set_title("Segmentation")
for axis in axes:
axis.axis("off")
plt.show()
```
## Notes
- [`Tipsv2DptForDensePrediction`] runs a single shared backbone forward pass and produces three outputs simultaneously: `predicted_depth`, `normals`, and `segmentation_logits`. Use it when you need all three tasks for the same image.
- [`Tipsv2DptForDepthEstimation`], [`Tipsv2DptForNormalEstimation`], and [`Tipsv2DptForSemanticSegmentation`] are single-task variants that discard the other heads. Use them in a pipeline or for inference on a single task.
## Tipsv2DptConfig
[[autodoc]] Tipsv2DptConfig
## Tipsv2DptImageProcessor
[[autodoc]] Tipsv2DptImageProcessor
- preprocess
- post_process_depth_estimation
- post_process_normal_estimation
- post_process_semantic_segmentation
## Tipsv2DptForDensePrediction
[[autodoc]] Tipsv2DptForDensePrediction
- forward
## Tipsv2DptForDepthEstimation
[[autodoc]] Tipsv2DptForDepthEstimation
- forward
## Tipsv2DptForNormalEstimation
[[autodoc]] Tipsv2DptForNormalEstimation
- forward
## Tipsv2DptForSemanticSegmentation
[[autodoc]] Tipsv2DptForSemanticSegmentation
- forward
