skeletal animation support added to API and software backend
SkCanvas::drawVertices now supports overloads that take an array of bone deformation matrices.
SkVertices::MakeCopy and SkVertices::Builder now support two additional optional attributes, boneIndices and boneWeights.
Bug: skia:
Change-Id: I30a3b11691e7cdb13924907cc1401ff86d127aea
Reviewed-on: https://skia-review.googlesource.com/137221
Reviewed-by: Brian Osman <brianosman@google.com>
Reviewed-by: Robert Phillips <robertphillips@google.com>
Commit-Queue: Ruiqi Mao <ruiqimao@google.com>
diff --git a/tools/viewer/NIMASlide.cpp b/tools/viewer/NIMASlide.cpp
new file mode 100644
index 0000000..c13ff2f
--- /dev/null
+++ b/tools/viewer/NIMASlide.cpp
@@ -0,0 +1,516 @@
+/*
+* Copyright 2018 Google Inc.
+*
+* Use of this source code is governed by a BSD-style license that can be
+* found in the LICENSE file.
+*/
+
+#include "NIMASlide.h"
+
+#include "SkAnimTimer.h"
+#include "SkOSPath.h"
+#include "Resources.h"
+#include "imgui.h"
+
+#include <algorithm>
+#include <cmath>
+
+using namespace sk_app;
+using namespace nima;
+
+// NIMA stores its matrices as 6 floats to represent translation and scale. This function takes
+// that format and converts it into a 3x3 matrix representation.
+static void nima_to_skmatrix(const float* nimaData, SkMatrix& matrix) {
+ matrix[0] = nimaData[0];
+ matrix[1] = nimaData[2];
+ matrix[2] = nimaData[4];
+ matrix[3] = nimaData[1];
+ matrix[4] = nimaData[3];
+ matrix[5] = nimaData[5];
+ matrix[6] = 0.0f;
+ matrix[7] = 0.0f;
+ matrix[8] = 1.0f;
+}
+
+// ImGui expects an array of const char* when displaying a ListBox. This function is for an
+// overload of ImGui::ListBox that takes a getter so that ListBox works with
+// std::vector<std::string>.
+static bool vector_getter(void* v, int index, const char** out) {
+ auto vector = reinterpret_cast<std::vector<std::string>*>(v);
+ *out = vector->at(index).c_str();
+ return true;
+}
+
+// A wrapper class that handles rendering of ActorImages (renderable components NIMA Actors).
+class NIMAActorImage {
+public:
+ NIMAActorImage(ActorImage* actorImage, SkImage* texture, SkPaint* paint)
+ : fActorImage(actorImage)
+ , fTexture(texture)
+ , fPaint(paint)
+ , fSkinned(false)
+ , fPositions()
+ , fTexs()
+ , fBoneIdx()
+ , fBoneWgt()
+ , fIndices()
+ , fBones()
+ , fVertices(nullptr)
+ , fRenderMode(kBackend_RenderMode) {
+ // Update the vertices and bones.
+ this->updateVertices();
+ this->updateBones();
+
+ // Update the vertices object.
+ this->updateVerticesObject(false);
+ }
+
+ void renderBackend(SkCanvas* canvas) {
+ // Reset vertices if the render mode has changed.
+ if (fRenderMode != kBackend_RenderMode) {
+ fRenderMode = kBackend_RenderMode;
+ this->updateVertices();
+ this->updateVerticesObject(false);
+ }
+
+ canvas->save();
+
+ // Update the vertex data.
+ if (fActorImage->doesAnimationVertexDeform() && fActorImage->isVertexDeformDirty()) {
+ this->updateVertices();
+ this->updateVerticesObject(false);
+ fActorImage->isVertexDeformDirty(false);
+ }
+
+ // Update the bones.
+ this->updateBones();
+
+ // Draw the vertices object.
+ this->drawVerticesObject(canvas, true);
+
+ canvas->restore();
+ }
+
+ void renderImmediate(SkCanvas* canvas) {
+ // Reset vertices if the render mode has changed.
+ if (fRenderMode != kImmediate_RenderMode) {
+ fRenderMode = kImmediate_RenderMode;
+ this->updateVertices();
+ this->updateVerticesObject(true);
+ }
+
+ // Update the vertex data.
+ if (fActorImage->doesAnimationVertexDeform() && fActorImage->isVertexDeformDirty()) {
+ this->updateVertices();
+ fActorImage->isVertexDeformDirty(false);
+ }
+
+ // Update the vertices object.
+ this->updateVerticesObject(true);
+
+ // Draw the vertices object.
+ this->drawVerticesObject(canvas, false);
+ }
+
+ int drawOrder() const { return fActorImage->drawOrder(); }
+
+private:
+ void updateVertices() {
+ // Update whether the image is skinned.
+ fSkinned = fActorImage->connectedBoneCount() > 0;
+
+ // Retrieve data from the image.
+ uint32_t vertexCount = fActorImage->vertexCount();
+ uint32_t vertexStride = fActorImage->vertexStride();
+ float* vertexData = fActorImage->vertices();
+ uint32_t indexCount = fActorImage->triangleCount() * 3;
+ uint16_t* indexData = fActorImage->triangles();
+
+ // Don't render if not visible.
+ if (!vertexCount || fActorImage->textureIndex() < 0) {
+ fPositions.clear();
+ fTexs.clear();
+ fBoneIdx.clear();
+ fBoneWgt.clear();
+ fIndices.clear();
+ return;
+ }
+
+ // Split the vertex data.
+ fPositions.resize(vertexCount);
+ fTexs.resize(vertexCount);
+ fIndices.resize(indexCount);
+ if (fSkinned) {
+ fBoneIdx.resize(vertexCount * 4);
+ fBoneWgt.resize(vertexCount * 4);
+ }
+ for (uint32_t i = 0; i < vertexCount; i ++) {
+ uint32_t j = i * vertexStride;
+
+ // Get the attributes.
+ float* attrPosition = vertexData + j;
+ float* attrTex = vertexData + j + 2;
+ float* attrBoneIdx = vertexData + j + 4;
+ float* attrBoneWgt = vertexData + j + 8;
+
+ // Get deformed positions if necessary.
+ if (fActorImage->doesAnimationVertexDeform()) {
+ attrPosition = fActorImage->animationDeformedVertices() + i * 2;
+ }
+
+ // Set the data.
+ fPositions[i].set(attrPosition[0], attrPosition[1]);
+ fTexs[i].set(attrTex[0] * fTexture->width(), attrTex[1] * fTexture->height());
+ if (fSkinned) {
+ for (uint32_t k = 0; k < 4; k ++) {
+ fBoneIdx[i].indices[k] = static_cast<uint32_t>(attrBoneIdx[k]);
+ fBoneWgt[i].weights[k] = attrBoneWgt[k];
+ }
+ }
+ }
+ memcpy(fIndices.data(), indexData, indexCount * sizeof(uint16_t));
+ }
+
+ void updateBones() {
+ // NIMA matrices are a collection of 6 floats.
+ constexpr int kNIMAMatrixSize = 6;
+
+ // Set up the matrices for the first time.
+ if (fBones.size() == 0) {
+ int numMatrices = 1;
+ if (fSkinned) {
+ numMatrices = fActorImage->boneInfluenceMatricesLength() / kNIMAMatrixSize;
+ }
+ fBones.assign(numMatrices, SkMatrix());
+ }
+
+ if (fSkinned) {
+ // Update the matrices.
+ float* matrixData = fActorImage->boneInfluenceMatrices();
+ for (uint32_t i = 1; i < fBones.size(); i ++) {
+ SkMatrix& matrix = fBones[i];
+ float* data = matrixData + i * kNIMAMatrixSize;
+ nima_to_skmatrix(data, matrix);
+ }
+ }
+
+ // Set the zero matrix to be the world transform.
+ nima_to_skmatrix(fActorImage->worldTransform().values(), fBones[0]);
+ }
+
+ void updateVerticesObject(bool applyDeforms) {
+ std::vector<SkPoint>* positions = &fPositions;
+
+ // Apply deforms if requested.
+ uint32_t vertexCount = fPositions.size();
+ std::vector<SkPoint> deformedPositions;
+ if (applyDeforms) {
+ positions = &deformedPositions;
+ deformedPositions.reserve(vertexCount);
+ for (uint32_t i = 0; i < vertexCount; i ++) {
+ Vec2D nimaPoint(fPositions[i].x(), fPositions[i].y());
+ uint32_t* boneIdx = nullptr;
+ float* boneWgt = nullptr;
+ if (fSkinned) {
+ boneIdx = fBoneIdx[i].indices;
+ boneWgt = fBoneWgt[i].weights;
+ }
+ nimaPoint = this->deform(nimaPoint, boneIdx, boneWgt);
+ deformedPositions.push_back(SkPoint::Make(nimaPoint[0], nimaPoint[1]));
+ }
+ }
+
+ // Update the vertices object.
+ fVertices = SkVertices::MakeCopy(SkVertices::kTriangles_VertexMode,
+ vertexCount,
+ positions->data(),
+ fTexs.data(),
+ nullptr,
+ fBoneIdx.data(),
+ fBoneWgt.data(),
+ fIndices.size(),
+ fIndices.data());
+ }
+
+ void drawVerticesObject(SkCanvas* canvas, bool useBones) const {
+ // Determine the blend mode.
+ SkBlendMode blendMode;
+ switch (fActorImage->blendMode()) {
+ case BlendMode::Off: {
+ blendMode = SkBlendMode::kSrc;
+ break;
+ }
+ case BlendMode::Normal: {
+ blendMode = SkBlendMode::kSrcOver;
+ break;
+ }
+ case BlendMode::Additive: {
+ blendMode = SkBlendMode::kPlus;
+ break;
+ }
+ case BlendMode::Multiply: {
+ blendMode = SkBlendMode::kMultiply;
+ break;
+ }
+ case BlendMode::Screen: {
+ blendMode = SkBlendMode::kScreen;
+ break;
+ }
+ }
+
+ // Set the opacity.
+ fPaint->setAlpha(static_cast<U8CPU>(fActorImage->renderOpacity() * 255));
+
+ // Draw the vertices.
+ if (useBones) {
+ canvas->drawVertices(fVertices, fBones.data(), fBones.size(), blendMode, *fPaint);
+ } else {
+ canvas->drawVertices(fVertices, blendMode, *fPaint);
+ }
+
+ // Reset the opacity.
+ fPaint->setAlpha(255);
+ }
+
+ Vec2D deform(const Vec2D& position, uint32_t* boneIdx, float* boneWgt) const {
+ float px = position[0], py = position[1];
+ float px2 = px, py2 = py;
+ float influence[6] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
+
+ // Apply the world transform.
+ Mat2D worldTransform = fActorImage->worldTransform();
+ px2 = worldTransform[0] * px + worldTransform[2] * py + worldTransform[4];
+ py2 = worldTransform[1] * px + worldTransform[3] * py + worldTransform[5];
+
+ // Apply deformations based on bone offsets.
+ if (boneIdx && boneWgt) {
+ float* matrices = fActorImage->boneInfluenceMatrices();
+
+ for (uint32_t i = 0; i < 4; i ++) {
+ uint32_t index = boneIdx[i];
+ float weight = boneWgt[i];
+ for (int j = 0; j < 6; j ++) {
+ influence[j] += matrices[index * 6 + j] * weight;
+ }
+ }
+
+ px = influence[0] * px2 + influence[2] * py2 + influence[4];
+ py = influence[1] * px2 + influence[3] * py2 + influence[5];
+ } else {
+ px = px2;
+ py = py2;
+ }
+
+ // Return the transformed position.
+ return Vec2D(px, py);
+ }
+
+private:
+ ActorImage* fActorImage;
+ SkImage* fTexture;
+ SkPaint* fPaint;
+
+ bool fSkinned;
+ std::vector<SkPoint> fPositions;
+ std::vector<SkPoint> fTexs;
+ std::vector<SkVertices::BoneIndices> fBoneIdx;
+ std::vector<SkVertices::BoneWeights> fBoneWgt;
+ std::vector<uint16_t> fIndices;
+
+ std::vector<SkMatrix> fBones;
+ sk_sp<SkVertices> fVertices;
+
+ RenderMode fRenderMode;
+};
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Represents an Actor, or an animated character, in NIMA.
+class NIMAActor : public Actor {
+public:
+ NIMAActor(const std::string& basePath)
+ : fTexture(nullptr)
+ , fActorImages()
+ , fPaint()
+ , fAnimations() {
+ // Load the NIMA data.
+ std::string nimaPath((basePath + ".nima").c_str());
+ INHERITED::load(nimaPath);
+
+ // Load the image asset.
+ sk_sp<SkData> imageData = SkData::MakeFromFileName((basePath + ".png").c_str());
+ fTexture = SkImage::MakeFromEncoded(imageData);
+
+ // Create the paint.
+ fPaint.setShader(fTexture->makeShader(nullptr));
+ fPaint.setFilterQuality(SkFilterQuality::kLow_SkFilterQuality);
+
+ // Load the image nodes.
+ fActorImages.reserve(m_ImageNodeCount);
+ for (uint32_t i = 0; i < m_ImageNodeCount; i ++) {
+ fActorImages.emplace_back(m_ImageNodes[i], fTexture.get(), &fPaint);
+ }
+
+ // Sort the image nodes.
+ std::sort(fActorImages.begin(), fActorImages.end(), [](auto a, auto b) {
+ return a.drawOrder() < b.drawOrder();
+ });
+
+ // Get the list of animations.
+ fAnimations.reserve(m_AnimationsCount);
+ for (uint32_t i = 0; i < m_AnimationsCount; i ++) {
+ fAnimations.push_back(m_Animations[i].name());
+ }
+ }
+
+ void render(SkCanvas* canvas, RenderMode renderMode) {
+ // Render the image nodes.
+ for (auto& image : fActorImages) {
+ switch (renderMode) {
+ case kBackend_RenderMode: {
+ // Render with Skia backend.
+ image.renderBackend(canvas);
+ break;
+ }
+ case kImmediate_RenderMode: {
+ // Render with immediate backend.
+ image.renderImmediate(canvas);
+ break;
+ }
+ }
+ }
+ }
+
+ const std::vector<std::string>& getAnimations() const {
+ return fAnimations;
+ }
+
+private:
+ sk_sp<SkImage> fTexture;
+ std::vector<NIMAActorImage> fActorImages;
+ SkPaint fPaint;
+ std::vector<std::string> fAnimations;
+
+ typedef Actor INHERITED;
+};
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+NIMASlide::NIMASlide(const SkString& name, const SkString& path)
+ : fBasePath()
+ , fActor(nullptr)
+ , fPlaying(true)
+ , fTime(0.0f)
+ , fRenderMode(kBackend_RenderMode)
+ , fAnimation(nullptr)
+ , fAnimationIndex(0) {
+ fName = name;
+
+ // Get the path components.
+ SkString baseName = SkOSPath::Basename(path.c_str());
+ baseName.resize(baseName.size() - 5);
+ SkString dirName = SkOSPath::Dirname(path.c_str());
+ SkString basePath = SkOSPath::Join(dirName.c_str(), baseName.c_str());
+
+ // Save the base path.
+ fBasePath = std::string(basePath.c_str());
+}
+
+NIMASlide::~NIMASlide() {}
+
+void NIMASlide::draw(SkCanvas* canvas) {
+ canvas->save();
+
+ canvas->translate(500, 500);
+ canvas->scale(1, -1);
+
+ // Render the actor.
+ fActor->render(canvas, fRenderMode);
+
+ canvas->restore();
+
+ // Render the GUI.
+ this->renderGUI();
+}
+
+void NIMASlide::load(SkScalar winWidth, SkScalar winHeight) {
+ this->resetActor();
+}
+
+void NIMASlide::unload() {
+ // Discard resources.
+ fAnimation = nullptr;
+ fActor.reset(nullptr);
+}
+
+bool NIMASlide::animate(const SkAnimTimer& timer) {
+ // Apply the animation.
+ if (fAnimation) {
+ if (fPlaying) {
+ fTime = std::fmod(timer.secs(), fAnimation->max());
+ }
+ fAnimation->time(fTime);
+ fAnimation->apply(1.0f);
+ }
+ return true;
+}
+
+bool NIMASlide::onChar(SkUnichar c) {
+ return false;
+}
+
+bool NIMASlide::onMouse(SkScalar x, SkScalar y, Window::InputState state, uint32_t modifiers) {
+ return false;
+}
+
+void NIMASlide::resetActor() {
+ // Create the actor.
+ fActor = std::make_unique<NIMAActor>(fBasePath);
+
+ // Get the animation.
+ fAnimation = fActor->animationInstance(fActor->getAnimations()[fAnimationIndex]);
+}
+
+void NIMASlide::renderGUI() {
+ ImGui::SetNextWindowSize(ImVec2(300, 220));
+ ImGui::Begin("NIMA");
+
+ // List of animations.
+ auto animations = const_cast<std::vector<std::string>&>(fActor->getAnimations());
+ ImGui::PushItemWidth(-1);
+ if (ImGui::ListBox("Animations",
+ &fAnimationIndex,
+ vector_getter,
+ reinterpret_cast<void*>(&animations),
+ animations.size(),
+ 5)) {
+ resetActor();
+ }
+
+ // Playback control.
+ ImGui::Spacing();
+ if (ImGui::Button("Play")) {
+ fPlaying = true;
+ }
+ ImGui::SameLine();
+ if (ImGui::Button("Pause")) {
+ fPlaying = false;
+ }
+
+ // Time slider.
+ ImGui::PushItemWidth(-1);
+ ImGui::SliderFloat("Time", &fTime, 0.0f, fAnimation->max(), "Time: %.3f");
+
+ // Backend control.
+ int renderMode = fRenderMode;
+ ImGui::Spacing();
+ ImGui::RadioButton("Skia Backend", &renderMode, 0);
+ ImGui::RadioButton("Immediate Backend", &renderMode, 1);
+ if (renderMode == 0) {
+ fRenderMode = kBackend_RenderMode;
+ } else {
+ fRenderMode = kImmediate_RenderMode;
+ }
+
+ ImGui::End();
+}
diff --git a/tools/viewer/NIMASlide.h b/tools/viewer/NIMASlide.h
new file mode 100644
index 0000000..d19d101
--- /dev/null
+++ b/tools/viewer/NIMASlide.h
@@ -0,0 +1,59 @@
+/*
+* Copyright 2018 Google Inc.
+*
+* Use of this source code is governed by a BSD-style license that can be
+* found in the LICENSE file.
+*/
+
+#ifndef NIMASlide_DEFINED
+#define NIMASlide_DEFINED
+
+#include "Slide.h"
+
+#include "SkCanvas.h"
+#include "SkVertices.h"
+#include <nima/Actor.hpp>
+#include <nima/ActorImage.hpp>
+#include <nima/Animation/ActorAnimationInstance.hpp>
+#include <nima/Vec2D.hpp>
+
+class NIMAActor;
+class NIMAActorImage;
+
+enum RenderMode {
+ kBackend_RenderMode = 0,
+ kImmediate_RenderMode = 1,
+};
+
+class NIMASlide : public Slide {
+public:
+ NIMASlide(const SkString& name, const SkString& path);
+ ~NIMASlide() override;
+
+ void draw(SkCanvas* canvas) override;
+ void load(SkScalar winWidth, SkScalar winHeight) override;
+ void unload() override;
+ bool animate(const SkAnimTimer& timer) override;
+
+ bool onChar(SkUnichar c) override;
+ bool onMouse(SkScalar x, SkScalar y, sk_app::Window::InputState state,
+ uint32_t modifiers) override;
+
+private:
+ void resetActor();
+
+ void renderGUI();
+
+private:
+ std::string fBasePath;
+ std::unique_ptr<NIMAActor> fActor;
+
+ bool fPlaying;
+ float fTime;
+ RenderMode fRenderMode;
+
+ nima::ActorAnimationInstance* fAnimation;
+ int fAnimationIndex;
+};
+
+#endif
diff --git a/tools/viewer/Viewer.cpp b/tools/viewer/Viewer.cpp
index 81c1851..27c8819 100644
--- a/tools/viewer/Viewer.cpp
+++ b/tools/viewer/Viewer.cpp
@@ -52,6 +52,10 @@
#include "SkottieSlide.h"
#endif
+#if !(defined(SK_BUILD_FOR_WIN) && defined(__clang__))
+ #include "NIMASlide.h"
+#endif
+
using namespace sk_app;
static std::map<GpuPathRenderers, std::string> gPathRendererNames;
@@ -565,6 +569,12 @@
[](const SkString& name, const SkString& path) -> sk_sp<Slide> {
return sk_make_sp<SvgSlide>(name, path);}
},
+#if !(defined(SK_BUILD_FOR_WIN) && defined(__clang__))
+ { ".nima", "nima-dir", FLAGS_nimas,
+ [](const SkString& name, const SkString& path) -> sk_sp<Slide> {
+ return sk_make_sp<NIMASlide>(name, path);}
+ },
+#endif
};
SkTArray<sk_sp<Slide>, true> dirSlides;