diff options
author | Ruiqi Mao <ruiqimao@google.com> | 2018-06-29 14:32:21 -0400 |
---|---|---|
committer | Skia Commit-Bot <skia-commit-bot@chromium.org> | 2018-06-29 19:34:28 +0000 |
commit | f510149da8d32f60f08d0a809eb037496079af3c (patch) | |
tree | 13b3670a481f3bae652968165d04f9ca4890d9bf /tools | |
parent | 38f118a2e7f986b06d69d0af41ec2d1af53dac39 (diff) |
skeletal animation support added to API and software backend
SkCanvas::drawVertices now supports overloads that take an array of bone deformation matrices.
SkVertices::MakeCopy and SkVertices::Builder now support two additional optional attributes, boneIndices and boneWeights.
Bug: skia:
Change-Id: I30a3b11691e7cdb13924907cc1401ff86d127aea
Reviewed-on: https://skia-review.googlesource.com/137221
Reviewed-by: Brian Osman <brianosman@google.com>
Reviewed-by: Robert Phillips <robertphillips@google.com>
Commit-Queue: Ruiqi Mao <ruiqimao@google.com>
Diffstat (limited to 'tools')
-rw-r--r-- | tools/debugger/SkDebugCanvas.cpp | 5 | ||||
-rw-r--r-- | tools/debugger/SkDebugCanvas.h | 3 | ||||
-rw-r--r-- | tools/flags/SkCommonFlags.cpp | 2 | ||||
-rw-r--r-- | tools/flags/SkCommonFlags.h | 1 | ||||
-rw-r--r-- | tools/viewer/NIMASlide.cpp | 516 | ||||
-rw-r--r-- | tools/viewer/NIMASlide.h | 59 | ||||
-rw-r--r-- | tools/viewer/Viewer.cpp | 10 |
7 files changed, 593 insertions, 3 deletions
diff --git a/tools/debugger/SkDebugCanvas.cpp b/tools/debugger/SkDebugCanvas.cpp index 8026eed479..537bcb1bf9 100644 --- a/tools/debugger/SkDebugCanvas.cpp +++ b/tools/debugger/SkDebugCanvas.cpp @@ -445,8 +445,9 @@ void SkDebugCanvas::onDrawPatch(const SkPoint cubics[12], const SkColor colors[4 this->addDrawCommand(new SkDrawPatchCommand(cubics, colors, texCoords, bmode, paint)); } -void SkDebugCanvas::onDrawVerticesObject(const SkVertices* vertices, SkBlendMode bmode, - const SkPaint& paint) { +void SkDebugCanvas::onDrawVerticesObject(const SkVertices* vertices, const SkMatrix* bones, + int boneCount, SkBlendMode bmode, const SkPaint& paint) { + // TODO: ANIMATION NOT LOGGED this->addDrawCommand(new SkDrawVerticesCommand(sk_ref_sp(const_cast<SkVertices*>(vertices)), bmode, paint)); } diff --git a/tools/debugger/SkDebugCanvas.h b/tools/debugger/SkDebugCanvas.h index c4a61e7887..b95614931b 100644 --- a/tools/debugger/SkDebugCanvas.h +++ b/tools/debugger/SkDebugCanvas.h @@ -146,7 +146,8 @@ protected: void onDrawArc(const SkRect&, SkScalar, SkScalar, bool, const SkPaint&) override; void onDrawRRect(const SkRRect&, const SkPaint&) override; void onDrawPoints(PointMode, size_t count, const SkPoint pts[], const SkPaint&) override; - void onDrawVerticesObject(const SkVertices*, SkBlendMode, const SkPaint&) override; + void onDrawVerticesObject(const SkVertices*, const SkMatrix* bones, int boneCount, SkBlendMode, + const SkPaint&) override; void onDrawPath(const SkPath&, const SkPaint&) override; void onDrawRegion(const SkRegion&, const SkPaint&) override; void onDrawBitmap(const SkBitmap&, SkScalar left, SkScalar top, const SkPaint*) override; diff --git a/tools/flags/SkCommonFlags.cpp b/tools/flags/SkCommonFlags.cpp index 73b0eb4f12..ed4d169585 100644 --- a/tools/flags/SkCommonFlags.cpp +++ b/tools/flags/SkCommonFlags.cpp @@ -57,10 +57,12 @@ DEFINE_bool(disableDriverCorrectnessWorkarounds, false, "Disables all GPU driver DEFINE_string(skps, "/data/local/tmp/skps", "Directory to read skps from."); DEFINE_string(jpgs, "/data/local/tmp/resources", "Directory to read jpgs from."); DEFINE_string(jsons, "/data/local/tmp/jsons", "Directory to read (Bodymovin) jsons from."); +DEFINE_string(nimas, "/data/local/tmp/nimas", "Directory to read NIMA animations from."); #else DEFINE_string(skps, "skps", "Directory to read skps from."); DEFINE_string(jpgs, "jpgs", "Directory to read jpgs from."); DEFINE_string(jsons, "jsons", "Directory to read (Bodymovin) jsons from."); +DEFINE_string(nimas, "nimas", "Directory to read NIMA animations from."); #endif DEFINE_int32(skpViewportSize, 1000, "Width & height of the viewport used to crop skp rendering."); diff --git a/tools/flags/SkCommonFlags.h b/tools/flags/SkCommonFlags.h index 3dca16f36d..10f90c0293 100644 --- a/tools/flags/SkCommonFlags.h +++ b/tools/flags/SkCommonFlags.h @@ -28,6 +28,7 @@ DECLARE_int32(skpViewportSize); DECLARE_string(jpgs); DECLARE_string(jsons); DECLARE_string(svgs); +DECLARE_string(nimas); DECLARE_bool(nativeFonts); DECLARE_int32(threads); DECLARE_string(resourcePath); diff --git a/tools/viewer/NIMASlide.cpp b/tools/viewer/NIMASlide.cpp new file mode 100644 index 0000000000..c13ff2f11d --- /dev/null +++ b/tools/viewer/NIMASlide.cpp @@ -0,0 +1,516 @@ +/* +* Copyright 2018 Google Inc. +* +* Use of this source code is governed by a BSD-style license that can be +* found in the LICENSE file. +*/ + +#include "NIMASlide.h" + +#include "SkAnimTimer.h" +#include "SkOSPath.h" +#include "Resources.h" +#include "imgui.h" + +#include <algorithm> +#include <cmath> + +using namespace sk_app; +using namespace nima; + +// NIMA stores its matrices as 6 floats to represent translation and scale. This function takes +// that format and converts it into a 3x3 matrix representation. +static void nima_to_skmatrix(const float* nimaData, SkMatrix& matrix) { + matrix[0] = nimaData[0]; + matrix[1] = nimaData[2]; + matrix[2] = nimaData[4]; + matrix[3] = nimaData[1]; + matrix[4] = nimaData[3]; + matrix[5] = nimaData[5]; + matrix[6] = 0.0f; + matrix[7] = 0.0f; + matrix[8] = 1.0f; +} + +// ImGui expects an array of const char* when displaying a ListBox. This function is for an +// overload of ImGui::ListBox that takes a getter so that ListBox works with +// std::vector<std::string>. +static bool vector_getter(void* v, int index, const char** out) { + auto vector = reinterpret_cast<std::vector<std::string>*>(v); + *out = vector->at(index).c_str(); + return true; +} + +// A wrapper class that handles rendering of ActorImages (renderable components NIMA Actors). +class NIMAActorImage { +public: + NIMAActorImage(ActorImage* actorImage, SkImage* texture, SkPaint* paint) + : fActorImage(actorImage) + , fTexture(texture) + , fPaint(paint) + , fSkinned(false) + , fPositions() + , fTexs() + , fBoneIdx() + , fBoneWgt() + , fIndices() + , fBones() + , fVertices(nullptr) + , fRenderMode(kBackend_RenderMode) { + // Update the vertices and bones. + this->updateVertices(); + this->updateBones(); + + // Update the vertices object. + this->updateVerticesObject(false); + } + + void renderBackend(SkCanvas* canvas) { + // Reset vertices if the render mode has changed. + if (fRenderMode != kBackend_RenderMode) { + fRenderMode = kBackend_RenderMode; + this->updateVertices(); + this->updateVerticesObject(false); + } + + canvas->save(); + + // Update the vertex data. + if (fActorImage->doesAnimationVertexDeform() && fActorImage->isVertexDeformDirty()) { + this->updateVertices(); + this->updateVerticesObject(false); + fActorImage->isVertexDeformDirty(false); + } + + // Update the bones. + this->updateBones(); + + // Draw the vertices object. + this->drawVerticesObject(canvas, true); + + canvas->restore(); + } + + void renderImmediate(SkCanvas* canvas) { + // Reset vertices if the render mode has changed. + if (fRenderMode != kImmediate_RenderMode) { + fRenderMode = kImmediate_RenderMode; + this->updateVertices(); + this->updateVerticesObject(true); + } + + // Update the vertex data. + if (fActorImage->doesAnimationVertexDeform() && fActorImage->isVertexDeformDirty()) { + this->updateVertices(); + fActorImage->isVertexDeformDirty(false); + } + + // Update the vertices object. + this->updateVerticesObject(true); + + // Draw the vertices object. + this->drawVerticesObject(canvas, false); + } + + int drawOrder() const { return fActorImage->drawOrder(); } + +private: + void updateVertices() { + // Update whether the image is skinned. + fSkinned = fActorImage->connectedBoneCount() > 0; + + // Retrieve data from the image. + uint32_t vertexCount = fActorImage->vertexCount(); + uint32_t vertexStride = fActorImage->vertexStride(); + float* vertexData = fActorImage->vertices(); + uint32_t indexCount = fActorImage->triangleCount() * 3; + uint16_t* indexData = fActorImage->triangles(); + + // Don't render if not visible. + if (!vertexCount || fActorImage->textureIndex() < 0) { + fPositions.clear(); + fTexs.clear(); + fBoneIdx.clear(); + fBoneWgt.clear(); + fIndices.clear(); + return; + } + + // Split the vertex data. + fPositions.resize(vertexCount); + fTexs.resize(vertexCount); + fIndices.resize(indexCount); + if (fSkinned) { + fBoneIdx.resize(vertexCount * 4); + fBoneWgt.resize(vertexCount * 4); + } + for (uint32_t i = 0; i < vertexCount; i ++) { + uint32_t j = i * vertexStride; + + // Get the attributes. + float* attrPosition = vertexData + j; + float* attrTex = vertexData + j + 2; + float* attrBoneIdx = vertexData + j + 4; + float* attrBoneWgt = vertexData + j + 8; + + // Get deformed positions if necessary. + if (fActorImage->doesAnimationVertexDeform()) { + attrPosition = fActorImage->animationDeformedVertices() + i * 2; + } + + // Set the data. + fPositions[i].set(attrPosition[0], attrPosition[1]); + fTexs[i].set(attrTex[0] * fTexture->width(), attrTex[1] * fTexture->height()); + if (fSkinned) { + for (uint32_t k = 0; k < 4; k ++) { + fBoneIdx[i].indices[k] = static_cast<uint32_t>(attrBoneIdx[k]); + fBoneWgt[i].weights[k] = attrBoneWgt[k]; + } + } + } + memcpy(fIndices.data(), indexData, indexCount * sizeof(uint16_t)); + } + + void updateBones() { + // NIMA matrices are a collection of 6 floats. + constexpr int kNIMAMatrixSize = 6; + + // Set up the matrices for the first time. + if (fBones.size() == 0) { + int numMatrices = 1; + if (fSkinned) { + numMatrices = fActorImage->boneInfluenceMatricesLength() / kNIMAMatrixSize; + } + fBones.assign(numMatrices, SkMatrix()); + } + + if (fSkinned) { + // Update the matrices. + float* matrixData = fActorImage->boneInfluenceMatrices(); + for (uint32_t i = 1; i < fBones.size(); i ++) { + SkMatrix& matrix = fBones[i]; + float* data = matrixData + i * kNIMAMatrixSize; + nima_to_skmatrix(data, matrix); + } + } + + // Set the zero matrix to be the world transform. + nima_to_skmatrix(fActorImage->worldTransform().values(), fBones[0]); + } + + void updateVerticesObject(bool applyDeforms) { + std::vector<SkPoint>* positions = &fPositions; + + // Apply deforms if requested. + uint32_t vertexCount = fPositions.size(); + std::vector<SkPoint> deformedPositions; + if (applyDeforms) { + positions = &deformedPositions; + deformedPositions.reserve(vertexCount); + for (uint32_t i = 0; i < vertexCount; i ++) { + Vec2D nimaPoint(fPositions[i].x(), fPositions[i].y()); + uint32_t* boneIdx = nullptr; + float* boneWgt = nullptr; + if (fSkinned) { + boneIdx = fBoneIdx[i].indices; + boneWgt = fBoneWgt[i].weights; + } + nimaPoint = this->deform(nimaPoint, boneIdx, boneWgt); + deformedPositions.push_back(SkPoint::Make(nimaPoint[0], nimaPoint[1])); + } + } + + // Update the vertices object. + fVertices = SkVertices::MakeCopy(SkVertices::kTriangles_VertexMode, + vertexCount, + positions->data(), + fTexs.data(), + nullptr, + fBoneIdx.data(), + fBoneWgt.data(), + fIndices.size(), + fIndices.data()); + } + + void drawVerticesObject(SkCanvas* canvas, bool useBones) const { + // Determine the blend mode. + SkBlendMode blendMode; + switch (fActorImage->blendMode()) { + case BlendMode::Off: { + blendMode = SkBlendMode::kSrc; + break; + } + case BlendMode::Normal: { + blendMode = SkBlendMode::kSrcOver; + break; + } + case BlendMode::Additive: { + blendMode = SkBlendMode::kPlus; + break; + } + case BlendMode::Multiply: { + blendMode = SkBlendMode::kMultiply; + break; + } + case BlendMode::Screen: { + blendMode = SkBlendMode::kScreen; + break; + } + } + + // Set the opacity. + fPaint->setAlpha(static_cast<U8CPU>(fActorImage->renderOpacity() * 255)); + + // Draw the vertices. + if (useBones) { + canvas->drawVertices(fVertices, fBones.data(), fBones.size(), blendMode, *fPaint); + } else { + canvas->drawVertices(fVertices, blendMode, *fPaint); + } + + // Reset the opacity. + fPaint->setAlpha(255); + } + + Vec2D deform(const Vec2D& position, uint32_t* boneIdx, float* boneWgt) const { + float px = position[0], py = position[1]; + float px2 = px, py2 = py; + float influence[6] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + + // Apply the world transform. + Mat2D worldTransform = fActorImage->worldTransform(); + px2 = worldTransform[0] * px + worldTransform[2] * py + worldTransform[4]; + py2 = worldTransform[1] * px + worldTransform[3] * py + worldTransform[5]; + + // Apply deformations based on bone offsets. + if (boneIdx && boneWgt) { + float* matrices = fActorImage->boneInfluenceMatrices(); + + for (uint32_t i = 0; i < 4; i ++) { + uint32_t index = boneIdx[i]; + float weight = boneWgt[i]; + for (int j = 0; j < 6; j ++) { + influence[j] += matrices[index * 6 + j] * weight; + } + } + + px = influence[0] * px2 + influence[2] * py2 + influence[4]; + py = influence[1] * px2 + influence[3] * py2 + influence[5]; + } else { + px = px2; + py = py2; + } + + // Return the transformed position. + return Vec2D(px, py); + } + +private: + ActorImage* fActorImage; + SkImage* fTexture; + SkPaint* fPaint; + + bool fSkinned; + std::vector<SkPoint> fPositions; + std::vector<SkPoint> fTexs; + std::vector<SkVertices::BoneIndices> fBoneIdx; + std::vector<SkVertices::BoneWeights> fBoneWgt; + std::vector<uint16_t> fIndices; + + std::vector<SkMatrix> fBones; + sk_sp<SkVertices> fVertices; + + RenderMode fRenderMode; +}; + +////////////////////////////////////////////////////////////////////////////////////////////////// + +// Represents an Actor, or an animated character, in NIMA. +class NIMAActor : public Actor { +public: + NIMAActor(const std::string& basePath) + : fTexture(nullptr) + , fActorImages() + , fPaint() + , fAnimations() { + // Load the NIMA data. + std::string nimaPath((basePath + ".nima").c_str()); + INHERITED::load(nimaPath); + + // Load the image asset. + sk_sp<SkData> imageData = SkData::MakeFromFileName((basePath + ".png").c_str()); + fTexture = SkImage::MakeFromEncoded(imageData); + + // Create the paint. + fPaint.setShader(fTexture->makeShader(nullptr)); + fPaint.setFilterQuality(SkFilterQuality::kLow_SkFilterQuality); + + // Load the image nodes. + fActorImages.reserve(m_ImageNodeCount); + for (uint32_t i = 0; i < m_ImageNodeCount; i ++) { + fActorImages.emplace_back(m_ImageNodes[i], fTexture.get(), &fPaint); + } + + // Sort the image nodes. + std::sort(fActorImages.begin(), fActorImages.end(), [](auto a, auto b) { + return a.drawOrder() < b.drawOrder(); + }); + + // Get the list of animations. + fAnimations.reserve(m_AnimationsCount); + for (uint32_t i = 0; i < m_AnimationsCount; i ++) { + fAnimations.push_back(m_Animations[i].name()); + } + } + + void render(SkCanvas* canvas, RenderMode renderMode) { + // Render the image nodes. + for (auto& image : fActorImages) { + switch (renderMode) { + case kBackend_RenderMode: { + // Render with Skia backend. + image.renderBackend(canvas); + break; + } + case kImmediate_RenderMode: { + // Render with immediate backend. + image.renderImmediate(canvas); + break; + } + } + } + } + + const std::vector<std::string>& getAnimations() const { + return fAnimations; + } + +private: + sk_sp<SkImage> fTexture; + std::vector<NIMAActorImage> fActorImages; + SkPaint fPaint; + std::vector<std::string> fAnimations; + + typedef Actor INHERITED; +}; + +////////////////////////////////////////////////////////////////////////////////////////////////// + +NIMASlide::NIMASlide(const SkString& name, const SkString& path) + : fBasePath() + , fActor(nullptr) + , fPlaying(true) + , fTime(0.0f) + , fRenderMode(kBackend_RenderMode) + , fAnimation(nullptr) + , fAnimationIndex(0) { + fName = name; + + // Get the path components. + SkString baseName = SkOSPath::Basename(path.c_str()); + baseName.resize(baseName.size() - 5); + SkString dirName = SkOSPath::Dirname(path.c_str()); + SkString basePath = SkOSPath::Join(dirName.c_str(), baseName.c_str()); + + // Save the base path. + fBasePath = std::string(basePath.c_str()); +} + +NIMASlide::~NIMASlide() {} + +void NIMASlide::draw(SkCanvas* canvas) { + canvas->save(); + + canvas->translate(500, 500); + canvas->scale(1, -1); + + // Render the actor. + fActor->render(canvas, fRenderMode); + + canvas->restore(); + + // Render the GUI. + this->renderGUI(); +} + +void NIMASlide::load(SkScalar winWidth, SkScalar winHeight) { + this->resetActor(); +} + +void NIMASlide::unload() { + // Discard resources. + fAnimation = nullptr; + fActor.reset(nullptr); +} + +bool NIMASlide::animate(const SkAnimTimer& timer) { + // Apply the animation. + if (fAnimation) { + if (fPlaying) { + fTime = std::fmod(timer.secs(), fAnimation->max()); + } + fAnimation->time(fTime); + fAnimation->apply(1.0f); + } + return true; +} + +bool NIMASlide::onChar(SkUnichar c) { + return false; +} + +bool NIMASlide::onMouse(SkScalar x, SkScalar y, Window::InputState state, uint32_t modifiers) { + return false; +} + +void NIMASlide::resetActor() { + // Create the actor. + fActor = std::make_unique<NIMAActor>(fBasePath); + + // Get the animation. + fAnimation = fActor->animationInstance(fActor->getAnimations()[fAnimationIndex]); +} + +void NIMASlide::renderGUI() { + ImGui::SetNextWindowSize(ImVec2(300, 220)); + ImGui::Begin("NIMA"); + + // List of animations. + auto animations = const_cast<std::vector<std::string>&>(fActor->getAnimations()); + ImGui::PushItemWidth(-1); + if (ImGui::ListBox("Animations", + &fAnimationIndex, + vector_getter, + reinterpret_cast<void*>(&animations), + animations.size(), + 5)) { + resetActor(); + } + + // Playback control. + ImGui::Spacing(); + if (ImGui::Button("Play")) { + fPlaying = true; + } + ImGui::SameLine(); + if (ImGui::Button("Pause")) { + fPlaying = false; + } + + // Time slider. + ImGui::PushItemWidth(-1); + ImGui::SliderFloat("Time", &fTime, 0.0f, fAnimation->max(), "Time: %.3f"); + + // Backend control. + int renderMode = fRenderMode; + ImGui::Spacing(); + ImGui::RadioButton("Skia Backend", &renderMode, 0); + ImGui::RadioButton("Immediate Backend", &renderMode, 1); + if (renderMode == 0) { + fRenderMode = kBackend_RenderMode; + } else { + fRenderMode = kImmediate_RenderMode; + } + + ImGui::End(); +} diff --git a/tools/viewer/NIMASlide.h b/tools/viewer/NIMASlide.h new file mode 100644 index 0000000000..d19d10119e --- /dev/null +++ b/tools/viewer/NIMASlide.h @@ -0,0 +1,59 @@ +/* +* Copyright 2018 Google Inc. +* +* Use of this source code is governed by a BSD-style license that can be +* found in the LICENSE file. +*/ + +#ifndef NIMASlide_DEFINED +#define NIMASlide_DEFINED + +#include "Slide.h" + +#include "SkCanvas.h" +#include "SkVertices.h" +#include <nima/Actor.hpp> +#include <nima/ActorImage.hpp> +#include <nima/Animation/ActorAnimationInstance.hpp> +#include <nima/Vec2D.hpp> + +class NIMAActor; +class NIMAActorImage; + +enum RenderMode { + kBackend_RenderMode = 0, + kImmediate_RenderMode = 1, +}; + +class NIMASlide : public Slide { +public: + NIMASlide(const SkString& name, const SkString& path); + ~NIMASlide() override; + + void draw(SkCanvas* canvas) override; + void load(SkScalar winWidth, SkScalar winHeight) override; + void unload() override; + bool animate(const SkAnimTimer& timer) override; + + bool onChar(SkUnichar c) override; + bool onMouse(SkScalar x, SkScalar y, sk_app::Window::InputState state, + uint32_t modifiers) override; + +private: + void resetActor(); + + void renderGUI(); + +private: + std::string fBasePath; + std::unique_ptr<NIMAActor> fActor; + + bool fPlaying; + float fTime; + RenderMode fRenderMode; + + nima::ActorAnimationInstance* fAnimation; + int fAnimationIndex; +}; + +#endif diff --git a/tools/viewer/Viewer.cpp b/tools/viewer/Viewer.cpp index 81c1851783..27c88191c3 100644 --- a/tools/viewer/Viewer.cpp +++ b/tools/viewer/Viewer.cpp @@ -52,6 +52,10 @@ #include "SkottieSlide.h" #endif +#if !(defined(SK_BUILD_FOR_WIN) && defined(__clang__)) + #include "NIMASlide.h" +#endif + using namespace sk_app; static std::map<GpuPathRenderers, std::string> gPathRendererNames; @@ -565,6 +569,12 @@ void Viewer::initSlides() { [](const SkString& name, const SkString& path) -> sk_sp<Slide> { return sk_make_sp<SvgSlide>(name, path);} }, +#if !(defined(SK_BUILD_FOR_WIN) && defined(__clang__)) + { ".nima", "nima-dir", FLAGS_nimas, + [](const SkString& name, const SkString& path) -> sk_sp<Slide> { + return sk_make_sp<NIMASlide>(name, path);} + }, +#endif }; SkTArray<sk_sp<Slide>, true> dirSlides; |