github
/
rife-ncnn-vulkan
mirror of https://github.com/nihui/rife-ncnn-vulkan
1
Fork 0
Code Releases Activity

wip

This commit is contained in:
nihui 2020-11-24 23:28:23 +08:00
parent d809a085cb
commit 79cb886954
7 changed files with 45 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitmodules vendored Normal file
View File

@ -0,0 +1,6 @@
[submodule "src/ncnn"]
path = src/ncnn
url = https://github.com/Tencent/ncnn.git
[submodule "src/libwebp"]
path = src/libwebp
url = https://github.com/webmproject/libwebp.git

4
models/rife/flownet.param
View File

@ -80,7 +80,7 @@ Interp Resize_94 1 1 461 471 0=2 1=4.000000e+00
Split splitncnn_13 1 5 471 471_splitncnn_0 471_splitncnn_1 471_splitncnn_2 471_splitncnn_3 471_splitncnn_4
Crop Slice_99 1 1 343_splitncnn_3 476 -23309=1,0 -23310=1,3 -23311=1,0
rife.Warp Warp_105 2 1 476 471_splitncnn_4 482
Crop Slice_110 1 1 343_splitncnn_2 487 -23309=1,3 -23310=1,-1 -23311=1,0
Crop Slice_110 1 1 343_splitncnn_2 487 -23309=1,3 -23310=1,2147483647 -23311=1,0
UnaryOp Neg_111 1 1 471_splitncnn_3 488 0=1
rife.Warp Warp_117 2 1 487 488 494
Concat Concat_118 3 1 482 494 471_splitncnn_2 495
@ -161,7 +161,7 @@ BinaryOp Add_212 2 1 471_splitncnn_1 623_splitn
Split splitncnn_27 1 3 624 624_splitncnn_0 624_splitncnn_1 624_splitncnn_2
Crop Slice_217 1 1 343_splitncnn_1 629 -23309=1,0 -23310=1,3 -23311=1,0
rife.Warp Warp_223 2 1 629 624_splitncnn_2 635
Crop Slice_228 1 1 343_splitncnn_0 640 -23309=1,3 -23310=1,-1 -23311=1,0
Crop Slice_228 1 1 343_splitncnn_0 640 -23309=1,3 -23310=1,2147483647 -23311=1,0
UnaryOp Neg_229 1 1 624_splitncnn_1 641 0=1
rife.Warp Warp_235 2 1 640 641 647
Concat Concat_236 3 1 635 647 624_splitncnn_0 648

42
models/rife/fusionnet.param
View File

@ -1,5 +1,5 @@
7767517
83 96
97 114
Input img0 0 1 img0
Input img1 0 1 img1
Input flow 0 1 flow
@ -13,57 +13,59 @@ Input 8 0 1 8
Input 9 0 1 9
Input 10 0 1 10
rife.Warp Warp_5 2 1 img0 flow_splitncnn_2 70
Split splitncnn_0 1 2 70 70_splitncnn_0 70_splitncnn_1
UnaryOp Neg_6 1 1 flow_splitncnn_1 71 0=1
rife.Warp Warp_12 2 1 img1 71 77
Concat Concat_13 3 1 70 77 flow_splitncnn_0 78
Split splitncnn_0 1 2 78 78_splitncnn_0 78_splitncnn_1
Split splitncnn_1 1 2 77 77_splitncnn_0 77_splitncnn_1
Concat Concat_13 3 1 70_splitncnn_1 77_splitncnn_1 flow_splitncnn_0 78
Split splitncnn_2 1 2 78 78_splitncnn_0 78_splitncnn_1
Convolution Conv_14 1 1 78_splitncnn_1 79 0=32 1=3 3=2 4=1 6=2304
Convolution Conv_15 1 1 78_splitncnn_0 80 0=32 1=3 3=2 4=1 5=1 6=2304
PReLU PRelu_16 1 1 80 82 0=32
Convolution Conv_17 1 1 82 83 0=32 1=3 4=1 5=1 6=9216
Split splitncnn_1 1 2 83 83_splitncnn_0 83_splitncnn_1
Split splitncnn_3 1 2 83 83_splitncnn_0 83_splitncnn_1
Pooling ReduceMean_19 1 1 83_splitncnn_1 85 0=1 4=1
InnerProduct Conv_20 1 1 85 88 0=16 2=512 9=2 -23310=1,8.258036e-02
InnerProduct Conv_22 1 1 88 90 0=32 2=512 9=4
BinaryOp Mul_24 2 1 83_splitncnn_0 90 91 0=2
BinaryOp Add_25 2 1 91 79 92
PReLU PRelu_26 1 1 92 94 0=32
Split splitncnn_2 1 2 94 94_splitncnn_0 94_splitncnn_1
Split splitncnn_4 1 2 94 94_splitncnn_0 94_splitncnn_1
Concat Concat_27 3 1 94_splitncnn_1 3 7 95
Split splitncnn_3 1 2 95 95_splitncnn_0 95_splitncnn_1
Split splitncnn_5 1 2 95 95_splitncnn_0 95_splitncnn_1
Convolution Conv_28 1 1 95_splitncnn_1 96 0=64 1=3 3=2 4=1 6=36864
Convolution Conv_29 1 1 95_splitncnn_0 97 0=64 1=3 3=2 4=1 5=1 6=36864
PReLU PRelu_30 1 1 97 99 0=64
Convolution Conv_31 1 1 99 100 0=64 1=3 4=1 5=1 6=36864
Split splitncnn_4 1 2 100 100_splitncnn_0 100_splitncnn_1
Split splitncnn_6 1 2 100 100_splitncnn_0 100_splitncnn_1
Pooling ReduceMean_33 1 1 100_splitncnn_1 102 0=1 4=1
InnerProduct Conv_34 1 1 102 105 0=16 2=1024 9=2 -23310=1,1.095001e-01
InnerProduct Conv_36 1 1 105 107 0=64 2=1024 9=4
BinaryOp Mul_38 2 1 100_splitncnn_0 107 108 0=2
BinaryOp Add_39 2 1 108 96 109
PReLU PRelu_40 1 1 109 111 0=64
Split splitncnn_5 1 2 111 111_splitncnn_0 111_splitncnn_1
Split splitncnn_7 1 2 111 111_splitncnn_0 111_splitncnn_1
Concat Concat_41 3 1 111_splitncnn_1 4 8 112
Split splitncnn_6 1 2 112 112_splitncnn_0 112_splitncnn_1
Split splitncnn_8 1 2 112 112_splitncnn_0 112_splitncnn_1
Convolution Conv_42 1 1 112_splitncnn_1 113 0=128 1=3 3=2 4=1 6=147456
Convolution Conv_43 1 1 112_splitncnn_0 114 0=128 1=3 3=2 4=1 5=1 6=147456
PReLU PRelu_44 1 1 114 116 0=128
Convolution Conv_45 1 1 116 117 0=128 1=3 4=1 5=1 6=147456
Split splitncnn_7 1 2 117 117_splitncnn_0 117_splitncnn_1
Split splitncnn_9 1 2 117 117_splitncnn_0 117_splitncnn_1
Pooling ReduceMean_47 1 1 117_splitncnn_1 119 0=1 4=1
InnerProduct Conv_48 1 1 119 122 0=16 2=2048 9=2 -23310=1,1.442167e-02
InnerProduct Conv_50 1 1 122 124 0=128 2=2048 9=4
BinaryOp Mul_52 2 1 117_splitncnn_0 124 125 0=2
BinaryOp Add_53 2 1 125 113 126
PReLU PRelu_54 1 1 126 128 0=128
Split splitncnn_8 1 2 128 128_splitncnn_0 128_splitncnn_1
Split splitncnn_10 1 2 128 128_splitncnn_0 128_splitncnn_1
Concat Concat_55 3 1 128_splitncnn_1 5 9 129
Split splitncnn_9 1 2 129 129_splitncnn_0 129_splitncnn_1
Split splitncnn_11 1 2 129 129_splitncnn_0 129_splitncnn_1
Convolution Conv_56 1 1 129_splitncnn_1 130 0=256 1=3 3=2 4=1 6=589824
Convolution Conv_57 1 1 129_splitncnn_0 131 0=256 1=3 3=2 4=1 5=1 6=589824
PReLU PRelu_58 1 1 131 133 0=256
Convolution Conv_59 1 1 133 134 0=256 1=3 4=1 5=1 6=589824
Split splitncnn_10 1 2 134 134_splitncnn_0 134_splitncnn_1
Split splitncnn_12 1 2 134 134_splitncnn_0 134_splitncnn_1
Pooling ReduceMean_61 1 1 134_splitncnn_1 136 0=1 4=1
InnerProduct Conv_62 1 1 136 139 0=16 2=4096 9=2 -23310=1,1.379933e-02
InnerProduct Conv_64 1 1 139 141 0=256 2=4096 9=4
@ -82,4 +84,16 @@ PReLU PRelu_77 1 1 155 157 0=32
Concat Concat_78 2 1 157 94_splitncnn_0 158
Deconvolution ConvTranspose_79 1 1 158 159 0=16 1=4 3=2 4=1 5=1 6=16384
PReLU PRelu_80 1 1 159 161 0=16
Convolution Conv_81 1 1 161 162 0=4 1=3 4=1 5=1 6=576
Convolution Conv_81 1 1 161 163 0=4 1=3 4=1 5=1 6=576 9=4
Split splitncnn_13 1 2 163 163_splitncnn_0 163_splitncnn_1
Crop Slice_87 1 1 163_splitncnn_1 168 -23309=1,0 -23310=1,3 -23311=1,0
BinaryOp Mul_89 1 1 168 170 0=2 1=1 2=2.000000e+00
BinaryOp Sub_91 1 1 170 172 0=1 1=1 2=1.000000e+00
Crop Slice_96 1 1 163_splitncnn_0 177 -23309=1,3 -23310=1,4 -23311=1,0
Split splitncnn_14 1 2 177 177_splitncnn_0 177_splitncnn_1
BinaryOp Mul_97 2 1 70_splitncnn_0 177_splitncnn_1 178 0=2
BinaryOp Sub_99 1 1 177_splitncnn_0 180 0=7 1=1 2=1.000000e+00
BinaryOp Mul_100 2 1 77_splitncnn_0 180 181 0=2
BinaryOp Add_101 2 1 178 181 182
BinaryOp Add_102 2 1 182 172 183
Clip Clip_103 1 1 183 188 0=0.000000e+00 1=1.000000e+00

10
src/CMakeLists.txt
View File

@ -95,7 +95,7 @@ if(NOT USE_SYSTEM_NCNN)
option(NCNN_PIXEL_ROTATE "" OFF)
option(NCNN_VULKAN "" ON)
option(NCNN_VULKAN_ONLINE_SPIRV "" ON)
option(NCNN_BUILD_BENCHMARK "" OFF)
option(NCNN_BUILD_BENCHMARK "" ON)
option(NCNN_BUILD_TESTS "" OFF)
option(NCNN_BUILD_TOOLS "" OFF)
option(NCNN_BUILD_EXAMPLES "" OFF)
@ -125,14 +125,14 @@ if(NOT USE_SYSTEM_NCNN)
option(WITH_LAYER_mvn "" OFF)
option(WITH_LAYER_pooling "" ON)
option(WITH_LAYER_power "" OFF)
option(WITH_LAYER_prelu "" OFF)
option(WITH_LAYER_prelu "" ON)
option(WITH_LAYER_proposal "" OFF)
option(WITH_LAYER_reduction "" OFF)
option(WITH_LAYER_relu "" ON)
option(WITH_LAYER_reshape "" OFF)
option(WITH_LAYER_roipooling "" OFF)
option(WITH_LAYER_scale "" OFF)
option(WITH_LAYER_sigmoid "" OFF)
option(WITH_LAYER_sigmoid "" ON)
option(WITH_LAYER_slice "" OFF)
option(WITH_LAYER_softmax "" OFF)
option(WITH_LAYER_split "" ON)
@ -156,8 +156,8 @@ if(NOT USE_SYSTEM_NCNN)
option(WITH_LAYER_deconvolutiondepthwise "" OFF)
option(WITH_LAYER_shufflechannel "" OFF)
option(WITH_LAYER_instancenorm "" OFF)
option(WITH_LAYER_clip "" OFF)
option(WITH_LAYER_reorg "" ON)
option(WITH_LAYER_clip "" ON)
option(WITH_LAYER_reorg "" OFF)
option(WITH_LAYER_yolodetectionoutput "" OFF)
option(WITH_LAYER_quantize "" OFF)
option(WITH_LAYER_dequantize "" OFF)

1
src/libwebp Submodule

@ -0,0 +1 @@
Subproject commit 5abb55823bb6196a918dd87202b2f32bbaff4c18

1
src/ncnn Submodule

@ -0,0 +1 @@
Subproject commit 6f49529da23cb7613670a4485a2bee549e9f7236

16
src/rife.cpp
View File

@ -311,9 +311,7 @@ int RIFE::process(const ncnn::Mat& in0image, const ncnn::Mat& in1image, float ti
}
// fusionnet
ncnn::VkMat warped_img0;
ncnn::VkMat warped_img1;
ncnn::VkMat refine_output;
ncnn::VkMat out_gpu_padded;
{
ncnn::Extractor ex = fusionnet.create_extractor();
ex.set_blob_vkallocator(blob_vkallocator);
@ -331,19 +329,9 @@ int RIFE::process(const ncnn::Mat& in0image, const ncnn::Mat& in1image, float ti
ex.input("8", ctx1[1]);
ex.input("9", ctx1[2]);
ex.input("10", ctx1[3]);
ex.extract("70", warped_img0, cmd);
ex.extract("77", warped_img1, cmd);
ex.extract("162", refine_output, cmd);
ex.extract("188", out_gpu_padded, cmd);
}
ncnn::VkMat out_gpu_padded;
// TODO implement this in postproc
// res = torch.sigmoid(refine_output[:, :3]) * 2 - 1
// mask = torch.sigmoid(refine_output[:, 3:4])
// merged_img = warped_img0 * mask + warped_img1 * (1 - mask)
// pred = merged_img + res
// pred = torch.clamp(pred, 0, 1)
// postproc
{
std::vector<ncnn::VkMat> bindings(2);