Creating Multi-image Panoramas in VisionOS

By Rotem Cohen

Abstract

It was very motivating to read Ron Brinkmann’s article about the possibilities of comics in VR and specifically the Vision Pro. After completing the necessary immediate features that Spalsh needed to be a fully fleshed out digital comic book reader, I set out to see his vision come to life, and above is the result, now available in v1.5.

I believe there’s a lot more to change and tweak, but since it took some looking and researching on my part, here are all the dots connected in one article, if you wish to recreate my panorama view approach, with or without combining images.

This approach would allow you to put any static SwiftUI view (not just images) on a panoramic curved plane that will scale according to your view’s size.

ViewModel variables

We’ll start by creating published variables in the viewModel that will store the image URLs of our pages and a boolean that is active when we’re in immersive panorama state:

// ViewModel.Swift
@Published var panoPages: [URL] = []
@Published var isPanoVisible: Bool = false

Create a new RealityView

This is the view that will host the curved plane. You can add attachments if you want to add buttons to get out of the view. I chose to utilize the view that calls the panorama for controls. For simplicity, I left that part out, but I might touch on it in future articles.

import SwiftUI
import RealityKit
import RealityKitContent

struct PanoView: View {
    @EnvironmentObject var viewModel: ViewModel
    
    var body: some View {
        RealityView { content in
            let panoEntity = ModelEntity()
            panoEntity.name = “Panorama”
            content.add(panoEntity)
        } update: { content  in
            
            // create new entity whenever panoPages updates
            if viewModel.panoPages.count > 0 {
                createPanoEntity(fileURLs: viewModel.panoPages, content: content)
            }
        }
    }
}

Creating the immersive Entity

The main method that combines creating the combined images’ texture, attach it to the curved plane mesh, and place it in correct position and orientation in front of the user:

private func createPanoEntity(fileURLs: [URL], content: RealityViewContent) {
   do {
      // combine images into one
      let combinedImages = saveImageTexture(fileURLs: fileURLs)

      // create a material with the combined images as its texture
      let material = createTexture(drawing: combinedImages)
            
      // calculate dimensions, so we’ll always have the same height, but different width
      let imageWidth = combinedImages.size.width
      let imageHeight = combinedImages.size.height
      let aspectRatio = Float(imageHeight / imageWidth)
            
            // create a curved plane mesh
      let PANO_HEIGHT : Float = 7.5
      let planeMesh = try generateCurvedPlane(width: PANO_HEIGHT / aspectRatio, depth: PANO_HEIGHT, vertices: (20,20), curveMagnitude: 1 + 1/aspectRatio)
            
      // find and update the entity
      var panoEntity = content.entities.first { entity in
          entity.name == "Panorama"
      }
      panoEntity!.components.set(
          ModelComponent(mesh: planeMesh, materials: [material])
      )
            
      // rotate the created plane so it faces the user
      let floorToWallQuat = simd_quatf(angle: Float.pi / 2, axis: simd_float3(x: 1, y: 0, z: 0))
      panoEntity!.setOrientation(floorToWallQuat, relativeTo: nil)
            
      // place the plane a bit away from the user and a little higher than the floor
      panoEntity!.setPosition(SIMD3<Float>(x: 0, y:1.5, z:-5.0), relativeTo: nil)
   } catch {
         print("Error creating panorama entity: \(error)")
   }
}

Converting a SwiftUI to an image

We use the method described here to convert a swiftUI view to an image

private func saveImageTexture(fileURLs: [URL])-> UIImage{
    let printedView = ImageView(fileURLs: fileURLs)
    let renderer = ImageRenderer(content: printedView)
    guard let image = renderer.uiImage else { return UIImage(imageLiteralResourceName: "Image") }
    return image
}

The very simple view that combines all images together. You can replace this with any static view to put practically anything in a curved plane:

@ViewBuilder
private func ImageView(fileURLs: [URL]) -> some View {
    HStack(spacing: 0) {
        ForEach(fileURLs, id:\.self) { url in
            if let uiImage = UIImage(contentsOfFile: url.path) {
                Image(uiImage: uiImage)
            }
        }
    }
}

Generating the texture:

private func createTexture(drawing: UIImage) -> UnlitMaterial {
   let texture = try! TextureResource.generate(from: drawing.cgImage!, options: .init(semantic: .normal)) 
   var material = UnlitMaterial()
   material.color = .init(texture: .init(texture))    
   return material
}   

Our method for creating a curved plane using code:

private func generateCurvedPlane(
    width: Float, depth: Float, vertices: (Int, Int), curveMagnitude: Float = 1.0
) throws -> MeshResource {
    var descr = MeshDescriptor()
    var meshPositions: [SIMD3<Float>] = []
    var indices: [UInt32] = []
    var textureMap: [SIMD2<Float>] = []
        
    for x_v in 0..<(vertices.0) {
       let vertexCounts = meshPositions.count
       for y_v in 0..<(vertices.1) {
           let xPosition = (Float(x_v) / Float(vertices.0 - 1) - 0.5) * width
           let zPosition = (0.5 - Float(y_v) / Float(vertices.1 - 1)) * depth
           let yPosition = -1 * (curveMagnitude - pow(xPosition, 2) * curveMagnitude / pow(width / 2, 2))
                
           meshPositions.append([xPosition, yPosition, zPosition])
           textureMap.append([Float(x_v) / Float(vertices.0 - 1), Float(y_v) / Float(vertices.1 - 1)])
                if x_v > 0 && y_v > 0 {
                    indices.append(
                        contentsOf: [
                            vertexCounts - vertices.1, vertexCounts, vertexCounts - vertices.1 + 1,
                            vertexCounts - vertices.1 + 1, vertexCounts, vertexCounts + 1
                        ].map { UInt32($0 + y_v - 1) })
                }
            }
        }
        
    descr.primitives = .triangles(indices)
    descr.positions = MeshBuffer(meshPositions)
    descr.textureCoordinates = MeshBuffers.TextureCoordinates(textureMap)
    return try .generate(from: [descr])
}

Add to App file

ImmersiveSpace(id: "ImmersiveSpace") {
    PanoView()
        .environmentObject(viewModel)
         .preferredSurroundingsEffect(.systemDark)
 }
 .immersionStyle(selection: .constant(viewModel.immersionStyle), in: .progressive, .mixed)
.windowStyle(.volumetric)
.defaultSize(width: 2.0, height: 2.0, depth: 2.0, in: .meters)

Open and close pano view methods

private func showPanoView(shownPages: Int) {
        Task {
            var images : [URL] = []
            for index in 0...shownPages - 1 {
                let currentIndex = Int(selectedPage) + index
                if currentIndex < imageFiles.count{
                    panoImages.append(imageFiles[currentIndex])
                }
            }
            viewModel.panoPages = images
            viewModel.isPanoVisible = true
            if viewModel.activeEnv == nil {
                await openImmersiveSpace(id: "ImmersiveSpace")
            }
        }
    }

private func closePanoView() {
    Task {
            openWindow(value: “Content”)
        }
        await dismissImmersiveSpace()
    }
}

That’s it!