[READ-ONLY] Mirror of https://github.com/flo-bit/image2fake3d. show a fake 3d effect in the browser from an image with a depth map
flo-bit.dev/image2fake3d/
3d-effect
depthmap
image3d
svelte
threlte
first try ios bug
+144
-124
+10
-103
src/lib/Convert.svelte
+10
-103
src/lib/Convert.svelte
···
1
1
<script lang="ts">
2
-
import { onMount } from "svelte";
3
-
import { Pydnet } from "./model";
4
-
import { currentImageIndex, customImages, isRunning, showMenu, useCustomImage } from "$lib/store";
5
-
6
2
let isDragging = false;
7
3
8
4
let input: HTMLInputElement;
···
14
10
const file = files?.[0];
15
11
16
12
if (file) {
17
-
runInference(file);
13
+
selectedFile(file);
18
14
}
19
15
}
20
16
···
36
32
}
37
33
}
38
34
39
-
let model: Pydnet | null = null;
35
+
export let selectedFile = (file: File) => {};
40
36
41
-
onMount(async () => {
42
-
if (!('createImageBitmap' in window)) {
43
-
// @ts-ignore
44
-
window.createImageBitmap = async function (data: ImageData) {
45
-
return new Promise((resolve, reject) => {
46
-
let dataURL;
47
-
const canvas = document.createElement('canvas');
48
-
const ctx = canvas.getContext('2d');
49
-
if(!ctx) throw new Error('Context not found');
50
-
canvas.width = data.width;
51
-
canvas.height = data.height;
52
-
ctx.putImageData(data, 0, 0);
53
-
dataURL = canvas.toDataURL();
54
-
55
-
const img = document.createElement('img');
56
-
img.addEventListener('load', function () {
57
-
resolve(this);
58
-
});
59
-
img.src = dataURL;
60
-
});
61
-
};
62
-
}
63
-
64
-
model = await new Pydnet().init();
65
-
});
66
-
67
-
async function runInference(file: File) {
68
-
var fr = new FileReader();
69
-
fr.onload = function () {
70
-
var img = new Image();
71
-
img.onload = function () {
72
-
display_result(img);
73
-
};
74
-
if(fr.result === null || typeof fr.result !== 'string') throw new Error('Result not found');
75
-
img.src = fr.result;
76
-
};
77
-
78
-
fr.readAsDataURL(file);
79
-
}
80
-
81
-
async function display_result(img: HTMLImageElement) {
82
-
if(!model) throw new Error('Model not loaded');
83
-
var results = await model.predict(img);
84
-
85
-
let canvas = document.createElement('canvas');
86
-
let ctx = canvas.getContext('2d');
87
-
canvas.width = img.width;
88
-
canvas.height = img.height;
89
-
90
-
let inputCanvas = document.createElement('canvas');
91
-
let inputCtx = inputCanvas.getContext('2d');
92
-
inputCanvas.width = img.width;
93
-
inputCanvas.height = img.height;
94
-
if(inputCtx) inputCtx.drawImage(img, 0, 0);
95
-
96
-
let buffer = new Uint8ClampedArray(model.width * model.height * 4);
97
-
let i = 0;
98
-
for (let y = 0; y < model.height; y++) {
99
-
for (let x = 0; x < model.width; x++) {
100
-
let index = y * model.width + x;
101
-
let depth = results[0][index];
102
-
buffer[i] = depth;
103
-
buffer[i + 1] = depth;
104
-
buffer[i + 2] = depth;
105
-
buffer[i + 3] = 255.0;
106
-
i += 4;
107
-
}
108
-
}
109
-
110
-
const imageData = new ImageData(buffer, model.width, model.height);
111
-
const renderer = await createImageBitmap(imageData);
112
-
if(!ctx) throw new Error('Context not found');
113
-
ctx.drawImage(renderer, 0, 0, img.width, img.height);
114
-
115
-
setTimeout(function () {
116
-
$isRunning = true;
117
-
$showMenu = false;
118
-
119
-
$customImages = [
120
-
...$customImages,
121
-
{
122
-
image: inputCanvas.toDataURL(),
123
-
depth: canvas.toDataURL(),
124
-
}
125
-
];
126
-
$currentImageIndex = $customImages.length - 1;
127
-
128
-
$useCustomImage = true;
129
-
}, 350);
130
-
}
37
+
export let text = 'No file added';
131
38
</script>
132
39
133
40
<div>
134
41
<label
135
42
for={data?.innerText.toLowerCase()}
136
43
bind:this={data}
137
-
class="block text-lg my-2 font-medium leading-6 text-neutral-900 dark:text-white"
44
+
class="block text-lg my-2 font-medium leading-6 text-white"
138
45
><slot>Try it with your image</slot></label
139
46
>
140
47
<!-- svelte-ignore a11y-no-static-element-interactions -->
141
48
<div class="flex flex-col">
142
49
<button
143
50
type="button"
144
-
class="mt-2 flex flex-col justify-center items-center rounded-lg border border-dashed border-neutral-900/25 dark:border-neutral-100/25 px-1 py-10 relative overflow-hidden h-40 w-full hover:bg-black/5 dark:hover:bg-white/5 {isDragging
51
+
class="mt-2 flex flex-col justify-center items-center rounded-lg border border-dashed border-neutral-100/25 px-1 py-10 relative overflow-hidden h-40 w-full hover:bg-white/5 {isDragging
145
52
? 'bg-black/5 dark:bg-white/5'
146
53
: ''}"
147
54
on:dragover|preventDefault={handleDragOver}
···
162
69
clip-rule="evenodd"
163
70
/>
164
71
</svg>
165
-
<p class="text-sm leading-6 text-neutral-300 dark:text-neutral-400">No file added</p>
72
+
<p class="text-sm leading-6 text-neutral-300">{text}</p>
166
73
167
74
</button>
168
75
<div class="text-center">
169
-
<div class="mt-2 text-sm leading-6 text-neutral-600 dark:text-neutral-300">
76
+
<div class="mt-2 text-sm flex leading-6 text-neutral-300 dark:text-neutral-300 justify-center">
170
77
<label
171
78
for={data?.innerText.toLowerCase()}
172
-
class="relative cursor-pointer rounded-md font-semibold text-accent-600 dark:text-accent-500 focus-within:outline-none focus-within:ring-2 focus-within:ring-accent-600 focus-within:ring-offset-2 hover:text-accent-500 dark:hover:text-accent-400"
79
+
class="relative cursor-pointer rounded-md font-semibold text-neutral-200 dark:text-neutral-300 focus-within:outline-none focus-within:ring-2 focus-within:ring-accent-600 focus-within:ring-offset-2 hover:text-accent-500 dark:hover:text-accent-400"
173
80
>
174
81
<button type="button" on:click={() => input.click()}>Browse files</button>
175
82
···
184
91
/>
185
92
</label>
186
93
187
-
<p class="">or drag and drop</p>
188
-
<p class="pt-2 text-xs">Your image stays on your device</p>
94
+
<p class="ml-1">or drag and drop</p>
189
95
</div>
96
+
<p class="pt-2 text-xs text-neutral-200">Your image stays on your device<br>On device depth estimation is currently broken for some images on iOS</p>
190
97
</div>
191
98
</div>
192
99
</div>
+134
-21
src/routes/+page.svelte
+134
-21
src/routes/+page.svelte
···
14
14
showMenu
15
15
} from '$lib/store';
16
16
17
+
import { Pydnet } from '$lib/model';
18
+
17
19
let useMotion = false;
18
20
let isCalibrated = false;
19
21
20
-
onMount(() => {
22
+
onMount(async () => {
21
23
// @ts-ignore
22
24
if (typeof DeviceOrientationEvent.requestPermission !== 'function') {
23
25
$isRunning = true;
24
26
}
27
+
28
+
if (!('createImageBitmap' in window)) {
29
+
// @ts-ignore
30
+
window.createImageBitmap = async function (data: ImageData) {
31
+
return new Promise((resolve, reject) => {
32
+
let dataURL;
33
+
const canvas = document.createElement('canvas');
34
+
const ctx = canvas.getContext('2d');
35
+
if (!ctx) throw new Error('Context not found');
36
+
canvas.width = data.width;
37
+
canvas.height = data.height;
38
+
ctx.putImageData(data, 0, 0);
39
+
dataURL = canvas.toDataURL();
40
+
41
+
const img = document.createElement('img');
42
+
img.addEventListener('load', function () {
43
+
resolve(this);
44
+
});
45
+
img.src = dataURL;
46
+
});
47
+
};
48
+
}
49
+
50
+
model = await new Pydnet().init();
25
51
});
26
52
27
53
function requestOrientationPermission() {
···
50
76
function onSwipe(event: CustomEvent) {
51
77
let direction = event.detail.direction;
52
78
if (direction === 'left') {
79
+
setImage($currentImageIndex + 1);
80
+
} else if (direction === 'right') {
53
81
setImage($currentImageIndex - 1);
54
82
}
55
-
if (direction === 'right') {
56
-
setImage($currentImageIndex + 1);
57
-
}
58
83
}
59
84
60
85
function setImage(i: number) {
···
83
108
}
84
109
85
110
let time = 0;
111
+
let model: Pydnet | null = null;
112
+
113
+
let imageDropperText = 'No file added';
114
+
115
+
async function runInference(file: File) {
116
+
imageDropperText = 'Processing file...';
117
+
var fr = new FileReader();
118
+
fr.onload = function () {
119
+
console.log('onload filereader');
120
+
var img = new Image();
121
+
img.onload = function () {
122
+
console.log('onload image');
123
+
display_result(img);
124
+
};
125
+
if (fr.result === null || typeof fr.result !== 'string') throw new Error('Result not found');
126
+
img.src = fr.result;
127
+
};
128
+
129
+
fr.readAsDataURL(file);
130
+
}
131
+
132
+
async function display_result(img: HTMLImageElement, tries = 0) {
133
+
if (!model) {
134
+
if (tries > 10) {
135
+
alert('This sadly doesnt work on your device :(');
136
+
return;
137
+
}
138
+
setTimeout(() => {
139
+
display_result(img, tries + 1);
140
+
}, 500);
141
+
return;
142
+
}
143
+
console.log('before predict');
144
+
var results = await model.predict(img);
145
+
console.log('after predict');
146
+
147
+
let canvas = document.createElement('canvas');
148
+
let ctx = canvas.getContext('2d');
149
+
canvas.width = img.width;
150
+
canvas.height = img.height;
151
+
152
+
let inputCanvas = document.createElement('canvas');
153
+
let inputCtx = inputCanvas.getContext('2d');
154
+
inputCanvas.width = img.width;
155
+
inputCanvas.height = img.height;
156
+
if (inputCtx) inputCtx.drawImage(img, 0, 0);
157
+
158
+
let buffer = new Uint8ClampedArray(model.width * model.height * 4);
159
+
let i = 0;
160
+
for (let y = 0; y < model.height; y++) {
161
+
for (let x = 0; x < model.width; x++) {
162
+
let index = y * model.width + x;
163
+
let depth = results[0][index];
164
+
buffer[i] = depth;
165
+
buffer[i + 1] = depth;
166
+
buffer[i + 2] = depth;
167
+
buffer[i + 3] = 255.0;
168
+
i += 4;
169
+
}
170
+
}
171
+
172
+
const imageData = new ImageData(buffer, model.width, model.height);
173
+
const renderer = await createImageBitmap(imageData);
174
+
if (!ctx) throw new Error('Context not found');
175
+
ctx.drawImage(renderer, 0, 0, img.width, img.height);
176
+
177
+
setTimeout(function () {
178
+
$isRunning = true;
179
+
$showMenu = false;
180
+
181
+
$customImages = [
182
+
...$customImages,
183
+
{
184
+
image: inputCanvas.toDataURL(),
185
+
depth: canvas.toDataURL()
186
+
}
187
+
];
188
+
$currentImageIndex = $customImages.length - 1;
189
+
190
+
$useCustomImage = true;
191
+
}, 350);
192
+
}
86
193
</script>
87
194
88
195
<svelte:window on:keyup={keyPressed} />
196
+
<div class="fixed inset-0 flex justify-center items-center text-neutral-200 -z-10">Loading...</div>
89
197
90
198
<div
91
199
class="w-screen h-screen"
···
109
217
<div class="overflow-scroll h-screen">
110
218
<div class="grid grid-cols-2 md:grid-cols-3 max-w-3xl p-4">
111
219
<div class="p-4 col-span-full">
112
-
<Convert />
220
+
<Convert selectedFile={runInference} text={imageDropperText} />
113
221
</div>
114
222
115
223
{#each $customImages as image, i}
···
139
247
</button>
140
248
{/each}
141
249
</div>
250
+
<div class="h-20"></div>
142
251
</div>
143
252
{/if}
144
253
</div>
···
146
255
{:else}
147
256
<div class="fixed bottom-0 text-neutral-200 w-full flex justify-center text-xs">
148
257
<div class="px-3 py-1.5 bg-black/50 font-semibold rounded-t-md text-center">
149
-
{useMotion ? 'tilt phone' : 'move mouse'} to see effect <br> {useMotion ? 'swipe' : 'arrow keys'} to change image
258
+
{useMotion ? 'tilt phone' : 'move mouse'} to see effect <br />
259
+
{useMotion ? 'swipe' : 'arrow keys'} to change image
150
260
</div>
151
261
</div>
152
262
<div class="fixed bottom-0 text-neutral-200 font-semibold w-full flex justify-between text-xs">
···
155
265
on:click={() => {
156
266
isCalibrated = false;
157
267
}}
158
-
class="px-3 py-1.5 bg-black/50 rounded-tr-md hover:bg-black/90 hover:text-neutral-400">Reset</button
268
+
class="px-3 py-1.5 bg-black/50 rounded-tr-md hover:bg-black/90 hover:text-neutral-400"
269
+
>Reset</button
159
270
>
160
271
{:else}
161
272
<div></div>
···
166
277
$showMenu = true;
167
278
$isRunning = false;
168
279
}}
169
-
class="px-3 py-1.5 bg-black/50 rounded-tl-md hover:bg-black/90 hover:text-neutral-400">Change</button
280
+
class="px-3 py-1.5 bg-black/50 rounded-tl-md hover:bg-black/90 hover:text-neutral-400"
281
+
>Change</button
170
282
>
171
283
</div>
172
284
173
285
<div class="fixed top-2 right-2 text-neutral-600 hover:text-neutral-400">
174
286
<a href="https://github.com/flo-bit/image2fake3d" target="_blank"
175
-
><svg fill="currentColor" viewBox="0 0 24 24" aria-hidden="true"
176
-
class="w-6 h-6">
287
+
><svg fill="currentColor" viewBox="0 0 24 24" aria-hidden="true" class="w-6 h-6">
177
288
<path
178
289
fill-rule="evenodd"
179
290
d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z"
···
184
295
</div>
185
296
{/if}
186
297
187
-
<Canvas>
188
-
{#key $currentImageIndex.toString() + $useCustomImage.toString()}
189
-
<Scene
190
-
bind:time
191
-
bind:isCalibrated
192
-
image={$useCustomImage
193
-
? $customImages[$currentImageIndex]
194
-
: presetImages[$currentImageIndex]}
195
-
/>
196
-
{/key}
197
-
</Canvas>
298
+
<div class="z-10 w-full h-full">
299
+
<Canvas>
300
+
{#key $currentImageIndex.toString() + $useCustomImage.toString()}
301
+
<Scene
302
+
bind:time
303
+
bind:isCalibrated
304
+
image={$useCustomImage
305
+
? $customImages[$currentImageIndex]
306
+
: presetImages[$currentImageIndex]}
307
+
/>
308
+
{/key}
309
+
</Canvas>
310
+
</div>
198
311
</div>