Matrix.hpp
#ifndef MATRIX_HPP
#define MATRIX_HPP
#include <vector>
#include <initializer_list>
#include<iostream>
#include <ostream>
#include <functional>
class Matrix{
public:
Matrix() = default;
Matrix(int rows,int cols,const std::initializer_list<float>&data){
rows_ = rows;
cols_ =cols;
data_=data;
if(data_.empty()){
data_.resize(rows*cols);
}else{
if(data_.size() != rows*cols ){
printf("Invalid construct.\n");
}
}
}
int rows() const{
return rows_;
}
int cols() const{
return cols_;
}
const std::vector<float> & data(){
return data_;
}
const float* ptr()const{
return data_.data();
}
float* ptr(){
return data_.data();
}
float& operator() (int ir, int ic){
int index = ir * cols_ + ic;
return data_[index];
}
const float& operator() (int ir, int ic)const{
int index = ir * cols_ + ic;
return data_[index];
}
Matrix element_wise(const std::function<float(float)>& func) const{
Matrix output(*this);
for(int i =0;i<output.data_.size();++i){
output.data_[i] = func(output.data_[i]);
}
return output;
}
Matrix operator*(const Matrix& value) const{
Matrix output(*this);
auto pleft = output.data_.data();
auto pright = value.data_.data();
for(int i=0; i<output.data_.size();++i){
*pleft++ *= *pright++;
}
return output;
}
Matrix operator*(float value) const{
return element_wise(
[&value](float x)->float
{
return x*value ;
}
);
}
Matrix operator+(float value) const{
return element_wise(
[&value](float x)->float
{
return x+value ;
}
);
}
Matrix operator-(float value) const{
return element_wise(
[&value](float x)->float
{
return x-value ;
}
);
}
Matrix operator/(float value) const{
return element_wise(
[&value](float x)->float
{
return x/value ;
}
);
}
Matrix gemm(const Matrix& other, bool ta ,bool tb,float alpha,float beta);
Matrix inverse(const Matrix& a);
private:
int rows_ = 0;
int cols_ = 0;
std::vector<float> data_;
};
std::ostream& operator<<(std::ostream& out, const Matrix& m){
printf("Matrix(%dx%d)\n",m.rows(),m.cols());
for(int ir = 0; ir < m.rows(); ++ir){
for(int ic = 0; ic < m.cols(); ++ic){
std::cout << m(ir,ic) << "\t";
}
std::cout<< "\n";
}
return out;
}
Matrix operator*(float value,const Matrix& m);
Matrix gemm(const Matrix& a,bool ta,const Matrix& b,bool tb, float alpha =1,0f,float beta =0.0f);
#endif
matrix.cpp
#include "matrix.hpp"
#include <openblas/cblas.h>
Matrix operator*(float value,const Matrix& m) {
return m*value;
}
Matrix Matrix::gemm(const Matrix& other, bool ta,bool tb ,float alpha,float beta){
return ::gemm(*this,ta,other,tb,alpha,beta);
}
Matrix gemm(const Matrix& a,bool ta,const Matrix& b,bool tb){
int ta_rows = ta?a.cols():a.rows();
int ta_cols = ta?a.rows():a.cols();
int tb_rows = tb?b.cols():b.rows();
int tb_cols = tb?b.rows():b.cols();
Matrix c(ta_rows,tb_cols);
int m = ta_rows;
int n = tb_cols;
int k = ta_cols;
int lda = a.cols();
int ldb = b.cols();
int ldc = c.cols();
cblas_segmm(
CblasRowMajor,
ta?CblasTrans:CblasNoTrans,
tb?CblasTrans:CblasNoTrans,
m,n,k,alpha,a.ptr(),lad,b.ptr(),ldb,beta,c.ptr(),ldc
)
return c;
Matrix inverse(const Matrix& a){
if(a.rows() != a.cols()){
printf("Invalid to compute inverse matrix by %d x %d\n", a.rows(), a.cols());
return Matrix();
}
Matrix output = a;
int n = a.rows();
int *ipiv = new int[n];
int code = LAPACKE_sgetrf(LAPACK_COL_MAJOR, n, n, output.ptr(), n, ipiv);
if(code == 0){
code = LAPACKE_sgetri(LAPACK_COL_MAJOR, n, output.ptr(), n, ipiv);
}
if(code != 0){
printf("LAPACKE inverse matrix failed, code = %d\n", code);
return Matrix();
}
delete[] ipiv;
return output;
}
autodiff.hpp
#ifndef AUTODIFF_HPP
#define AUTODIFF_HPP
#include <memory>
#include <vector>
#include <functional>
#include <random>
#include "matrix.hpp"
class ExpressionContainer{
public:
virtual const char* type() = 0;
virtual Matrix forward() = 0;
virtual void backward(const Matrix& gradient) = 0;
virtual bool requires_grad() const{
return requires_grad_;}
protected:
bool requires_grad_ = false;
};
class Parameter;
class Expression{
public:
Expression(){
};
Expression(const Matrix& value);
const char* type(){
return data_ ? data_->type() : "nullptr";}
Matrix forward();
void backward();
Expression view(const std::vector<int>& shape);
Expression power();
Expression gemm(const Expression& other);
std::shared_ptr<ExpressionContainer> data() const{
return data_;}
std::vector<Parameter> params() const{
return params_;};
protected:
std::shared_ptr<ExpressionContainer> data_;
std::vector<int> forward_output_shape_;
std::vector<Parameter> params_;
};
class Parameter : public Expression{
public:
Parameter();
Parameter(float value);
Parameter(const std::vector<float>& value);
Parameter(const Matrix& value);
const Matrix& gradient() const;
const Matrix& value() const;
Matrix& gradient();
Matrix& value();
operator Matrix&();
operator const Matrix&()const;
virtual bool requires_grad() const;
};
class Add : public Expression{
public:
Add();
Add& operator()(const Expression& left, const Expression& right);
};
class Sub : public Expression{
public:
Sub();
Sub& operator()(const Expression& left, const Expression& right);
};
class Multiply : public Expression{
public:
Multiply();
Multiply& operator()(const Expression& left, const Expression& right);
};
class View : public Expression{
public:
View(const std::vector<int>& shape);
View& operator()(const Expression& x);
};
class Power : public Expression{
public:
Power();
Power& operator()(const Expression& value);
};
class MatMul : public Expression{
public:
MatMul();
MatMul& operator()(const Expression& left, const Expression& right);
};
class Linear : public Expression{
public:
Linear(int input, int output, bool bias=true);
Linear& operator()(const Expression& x);
Parameter& weight();
Parameter& bias();
};
class Conv2d : public Expression{
public:
Conv2d(int input, int output, int ksize, int stride, int padding, bool bias=true);
Conv2d& operator()(const Expression& x);
Parameter& weight();
Parameter& bias();
};
class ReLU : public Expression{
public:
ReLU();
ReLU& operator()(const Expression& left);
};
class Sigmoid : public Expression{
public:
Sigmoid();
Sigmoid& operator()(const Expression& left);
};
class Log : public Expression{
public:
Log();
Log& operator()(const Expression& left);
};
class SigmoidCrossEntropyLoss : public Expression{
public:
SigmoidCrossEntropyLoss();
SigmoidCrossEntropyLoss& operator()(const Expression& predict, const Expression& label);
};
Expression operator+(const Expression& a, const Expression& b);
Expression operator+(float a, const Expression& b);
Expression operator+(const Expression& a, float b);
Expression operator-(const Expression& a, const Expression& b);
Expression operator-(float a, const Expression& b);
Expression operator-(const Expression& a, float b);
Expression operator*(const Expression& a, const Expression& b);
Expression operator*(float a, const Expression& b);
Expression operator*(const Expression& a, float b);
std::default_random_engine& get_random_engine();
Matrix create_normal_distribution_matrix(const std::vector<int>& shape, float mean=0.0f, float stddev=1.0f);
#endif
autodiff.cpp
#include "autodiff.hpp"
#include <stdio.h>
#include <string.h>
#include <cmath>
#include <random>
#include <iostream>
using namespace std;
default_random_engine& get_random_engine(){
static default_random_engine global_random_engine;
return global_random_engine;
}
Matrix create_normal_distribution_matrix(const vector<int>& shape, float mean, float stddev){
normal_distribution<float> norm(mean, stddev);
Matrix out(shape);
auto& engine = get_random_engine();
auto p = out.ptr();
for(int i = 0; i < out.numel(); ++i)
*p++ = norm(engine);
return out;
}
class MatrixContainer : public ExpressionContainer{
public:
MatrixContainer(const Matrix& value, bool requires_grad = false){
value_ = value;requires_grad_=requires_grad;}
virtual const char* type() override{
return "Matrix";}
virtual Matrix forward() override{
return value_;}
virtual void backward(const Matrix& gradient) override{
if(requires_grad_) gradient_ += gradient;}
const Matrix& gradient() const{
return gradient_;}
const Matrix& value() const{
return value_;}
Matrix& gradient(){
return gradient_;}
Matrix& value(){
return value_;}
private:
Matrix value_;
Matrix gradient_{
{
1, 1}, {
0}};
};
class AddContainer : public ExpressionContainer{
public:
void assign(const Expression& left, const Expression& right);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
std::shared_ptr<ExpressionContainer> left_;
std::shared_ptr<ExpressionContainer> right_;
int broadcast_ = 0;
};
class SubContainer : public ExpressionContainer{
public:
void assign(const Expression& left, const Expression& right);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
std::shared_ptr<ExpressionContainer> left_;
std::shared_ptr<ExpressionContainer> right_;
};
class MultiplyContainer : public ExpressionContainer{
public:
void assign(const Expression& left, const Expression& right);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
Matrix left_save_;
Matrix right_save_;
std::shared_ptr<ExpressionContainer> left_;
std::shared_ptr<ExpressionContainer> right_;
};
class ViewContainer : public ExpressionContainer{
public:
ViewContainer(const vector<int>& shape);
void assign(const Expression& value);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
std::vector<int> shape_, old_shape_;
std::shared_ptr<ExpressionContainer> value_;
};
class PowerContainer : public ExpressionContainer{
public:
void assign(const Expression& value);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
Matrix value_save_;
std::shared_ptr<ExpressionContainer> value_;
};
class MatMulContainer : public ExpressionContainer{
public:
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
void assign(const Expression& left, const Expression& right);
private:
Matrix left_save_;
Matrix right_save_;
std::shared_ptr<ExpressionContainer> left_;
std::shared_ptr<ExpressionContainer> right_;
};
class LinearContainer : public ExpressionContainer{
public:
LinearContainer(int input, int output, bool bias);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
void assign(const Expression& x);
Parameter& weight();
Parameter& bias();
private:
Matrix x_save_;
std::shared_ptr<ExpressionContainer> x_;
std::shared_ptr<Parameter> weight_, bias_;
int input_ = 0;
int output_ = 0;
bool hasbias_ = true;
};
class Conv2dContainer : public ExpressionContainer{
public:
Conv2dContainer(int input, int output, int ksize, int stride, int padding, bool bias);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
void assign(const Expression& x);
Parameter& weight();
Parameter& bias();
private:
std::vector<int> x_shape_;
Matrix column_, output_save_, grad_save_;
std::shared_ptr<ExpressionContainer> x_;
std::shared_ptr<Parameter> weight_, bias_;
int input_ = 0;
int output_ = 0;
int ksize_ = 0;
int stride_ = 0;
int padding_ = 0;
bool hasbias_ = true;
int oh_ = 0, ow_ = 0;
};
class ReLUContainer : public ExpressionContainer{
public:
void assign(const Expression& left);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
Matrix save_forward_;
std::shared_ptr<ExpressionContainer> left_;
};
class SigmoidContainer : public ExpressionContainer{
public:
void assign(const Expression& left);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
Matrix save_forward_;
std::shared_ptr<ExpressionContainer> left_;
};
class LogContainer : public ExpressionContainer{
public:
void assign(const Expression& left);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
Matrix save_forward_;
std::shared_ptr<ExpressionContainer> left_;
};
class SigmoidCrossEntropyLossContainer : public ExpressionContainer{
public:
void assign(const Expression& predict, const Expression& label);
virtual const char* type() override;
virtual Matrix forward() override;
virtual void backward(const Matrix& gradient) override;
private:
Matrix save_gradient_;
std::shared_ptr<ExpressionContainer> predict_;
std::shared_ptr<ExpressionContainer> label_;
};
Parameter::Parameter(){
data_.reset(new MatrixContainer(Matrix(), true));
}
Parameter::Parameter(float value){
data_.reset(new MatrixContainer(Matrix({
1, 1}, {
value}), true));
}
Parameter::Parameter(const vector<float>& value){
data_.reset(new MatrixContainer(Matrix({
(int)value.size(), 1}, value), true));
}
Parameter::Parameter(const Matrix& value){
data_.reset(new MatrixContainer(value, true));
}
bool Parameter::requires_grad() const{
return data_->requires_grad();
}
const Matrix& Parameter::gradient() const{
MatrixContainer* ptr = (MatrixContainer*)data_.get();
return ptr->gradient();
}
const Matrix& Parameter::value() const{
MatrixContainer* ptr = (MatrixContainer*)data_.get();
return ptr->value();
}
Matrix& Parameter::gradient(){
MatrixContainer* ptr = (MatrixContainer*)data_.get();
return ptr->gradient();
}
Parameter::operator Matrix&(){
MatrixContainer* ptr = (MatrixContainer*)data_.get();
return ptr->value();
}
Parameter::operator const Matrix&()const{
MatrixContainer* ptr = (MatrixContainer*)data_.get();
return ptr->value();
}
Matrix& Parameter::value(){
MatrixContainer* ptr = (MatrixContainer*)data_.get();
return ptr->value();
}
void AddContainer::assign(const Expression& left, const Expression& right){
left_ = left.data();
right_ = right.data();
requires_grad_ = left_->requires_grad() || right_->requires_grad();
}
const char* AddContainer::type(){
return "Add";
}
Matrix AddContainer::forward(){
auto lm = left_->forward();
auto rm = right_->forward();
Matrix* pa, *pb;
tie(pa, pb, broadcast_) = Matrix::check_broadcast(&lm, &rm);
if(pa != &lm) broadcast_ += 4;
return lm + rm;
}
void AddContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
Matrix g = gradient;
Matrix reduce = gradient;
if(broadcast_ == 1 || broadcast_ == 5){
reduce = g.reduce_sum_by_col();
}else if(broadcast_ == 2 || broadcast_ == 6){
reduce = g.reduce_sum_by_row();
}else if(broadcast_ == 3 || broadcast_ == 7){
reduce = g.reduce_sum_all();
}
if(broadcast_ > 3)
std::swap(g, reduce);
if(left_->requires_grad())
left_->backward(g);
if(right_->requires_grad())
right_->backward(reduce);
}
Add::Add(){
data_.reset(new AddContainer());
}
Add& Add::operator()(const Expression& left, const Expression& right){
std::dynamic_pointer_cast<AddContainer>(data_)->assign(left, right);
return *this;
}
void SubContainer::assign(const Expression& left, const Expression& right){
left_ = left.data();
right_ = right.data();
requires_grad_ = left_->requires_grad() || right_->requires_grad();
}
const char* SubContainer::type(){
return "Sub";
}
Matrix SubContainer::forward(){
return left_->forward() - right_->forward();
}
void SubContainer::backward(const Matrix& gradient){
if(left_->requires_grad())
left_->backward(gradient);
if(right_->requires_grad())
right_->backward(-gradient);
}
Sub::Sub(){
data_.reset(new SubContainer());
}
Sub& Sub::operator()(const Expression& left, const Expression& right){
std::dynamic_pointer_cast<SubContainer>(data_)->assign(left, right);
return *this;
}
void MultiplyContainer::assign(const Expression& left, const Expression& right){
left_ = left.data();
right_ = right.data();
requires_grad_ = left_->requires_grad() || right_->requires_grad();
}
const char* MultiplyContainer::type(){
return "Multiply";
}
Matrix MultiplyContainer::forward(){
left_save_ = left_->forward();
right_save_ = right_->forward();
return left_save_ * right_save_;
}
void MultiplyContainer::backward(const Matrix& gradient){
if(left_->requires_grad())
left_->backward(right_save_ * gradient);
if(right_->requires_grad())
right_->backward(left_save_ * gradient);
}
Multiply::Multiply(){
data_.reset(new MultiplyContainer());
}
Multiply& Multiply::operator()(const Expression& left, const Expression& right){
std::dynamic_pointer_cast<MultiplyContainer>(data_)->assign(left, right);
return *this;
}
ViewContainer::ViewContainer(const vector<int>& shape){
shape_ = shape;
}
void ViewContainer::assign(const Expression& value){
value_ = value.data();
requires_grad_ = value_->requires_grad();
}
const char* ViewContainer::type(){
return "View";
}
Matrix ViewContainer::forward(){
auto x = value_->forward();
old_shape_ = x.shape();
return x.view(shape_);
}
void ViewContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
value_->backward(gradient.view(old_shape_));
}
View::View(const vector<int>& shape){
data_.reset(new ViewContainer(shape));
}
View& View::operator()(const Expression& value){
std::dynamic_pointer_cast<ViewContainer>(data_)->assign(value);
return *this;
}
void PowerContainer::assign(const Expression& value){
value_ = value.data();
requires_grad_ = value_->requires_grad();
}
const char* PowerContainer::type(){
return "Power";
}
Matrix PowerContainer::forward(){
value_save_ = value_->forward();
return value_save_.power(2.0f);
}
void PowerContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
value_->backward(value_save_ * gradient * 2.0f);
}
Power::Power(){
data_.reset(new PowerContainer());
}
Power& Power::operator()(const Expression& value){
std::dynamic_pointer_cast<PowerContainer>(data_)->assign(value);
return *this;
}
void MatMulContainer::assign(const Expression& left, const Expression& right){
left_ = left.data();
right_ = right.data();
requires_grad_ = left_->requires_grad() || right_->requires_grad();
}
const char* MatMulContainer::type(){
return "MatMul";
}
Matrix MatMulContainer::forward(){
left_save_ = left_->forward();
right_save_ = right_->forward();
return left_save_.gemm(right_save_);
}
void MatMulContainer::backward(const Matrix& gradient){
if(left_->requires_grad())
left_->backward(gradient.gemm(right_save_, false, true));
if(right_->requires_grad())
right_->backward(left_save_.gemm(gradient, true));
}
MatMul::MatMul(){
data_.reset(new MatMulContainer());
}
MatMul& MatMul::operator()(const Expression& left, const Expression& right){
std::dynamic_pointer_cast<MatMulContainer>(data_)->assign(left, right);
return *this;
}
LinearContainer::LinearContainer(int input, int output, bool bias){
input_ = input;
output_ = output;
hasbias_ = bias;
float fan_in_fan_out = 1.0f / sqrt((float)(input_ + output_));
weight_.reset(new Parameter(create_normal_distribution_matrix({
input_, output_}, 0, fan_in_fan_out)));
if(hasbias_) bias_.reset(new Parameter(Matrix({
1, output_})));
}
Parameter& LinearContainer::weight(){
return *weight_.get();
}
Parameter& LinearContainer::bias(){
return *bias_.get();
}
void LinearContainer::assign(const Expression& x){
x_ = x.data();
requires_grad_ = true;
}
const char* LinearContainer::type(){
return "Linear";
}
Matrix LinearContainer::forward(){
x_save_ = x_->forward();
auto x = x_save_.gemm(weight_->value());
if(hasbias_)
x += bias_->value();
return x;
}
void LinearContainer::backward(const Matrix& gradient){
if(x_->requires_grad())
x_->backward(gradient.gemm(weight_->value(), false, true));
if(weight_->requires_grad())
weight_->data()->backward(x_save_.gemm(gradient, true));
if(hasbias_ && bias_->requires_grad())
bias_->data()->backward(gradient.reduce_sum_by_row());
}
Parameter& Linear::weight(){
return std::dynamic_pointer_cast<LinearContainer>(data_)->weight();
}
Parameter& Linear::bias(){
return std::dynamic_pointer_cast<LinearContainer>(data_)->bias();
}
Linear::Linear(int input, int output, bool bias){
data_.reset(new LinearContainer(input, output, bias));
params_ = {
this->weight(), this->bias()};
}
Linear& Linear::operator()(const Expression& x){
std::dynamic_pointer_cast<LinearContainer>(data_)->assign(x);
return *this;
}
Conv2dContainer::Conv2dContainer(int input, int output, int ksize, int stride, int padding, bool bias){
input_ = input;
output_ = output;
ksize_ = ksize;
stride_ = stride;
padding_ = padding;
hasbias_ = bias;
float fan_in_fan_out = 2.0f / sqrt((float)(input_ + output_));
weight_.reset(new Parameter(create_normal_distribution_matrix({
output_, input_, ksize_, ksize_}, 0, fan_in_fan_out)));
if(hasbias_) bias_.reset(new Parameter(
create_normal_distribution_matrix({
1, output_}, 0, 1)
));
}
Parameter& Conv2dContainer::weight(){
return *weight_.get();
}
Parameter& Conv2dContainer::bias(){
return *bias_.get();
}
void Conv2dContainer::assign(const Expression& x){
x_ = x.data();
requires_grad_ = true;
}
const char* Conv2dContainer::type(){
return "Conv2d";
}
Matrix Conv2dContainer::forward(){
auto xval = x_->forward();
x_shape_ = xval.shape();
int xb, xc, xh, xw;
xb = x_shape_[0];
xc = x_shape_[1];
xh = x_shape_[2];
xw = x_shape_[3];
ow_ = (x_shape_[3] + padding_ * 2 - ksize_) / stride_ + 1;
oh_ = (x_shape_[2] + padding_ * 2 - ksize_) / stride_ + 1;
int col_w = ow_ * oh_;
int col_h = ksize_ * ksize_ * input_;
column_.resize({
xb, col_h, col_w});
output_save_.resize({
xb, output_, oh_, ow_});
for(int ib = 0; ib < xb; ++ib){
auto subcolumn = column_.reference_d0(ib);
for(int ic = 0; ic < xc; ++ic){
for(int oy = 0; oy < oh_; ++oy){
for(int ox = 0; ox < ow_; ++ox){
int column_x = ox + oy * ow_;
for(int ky = 0; ky < ksize_; ++ky){
int column_idx = (ky + ic * ksize_) * ksize_;
int iy = oy * stride_ + ky - padding_;
if(iy < 0 || iy >= xh) continue;
for(int kx = 0; kx < ksize_; ++kx){
int column_y = column_idx + kx;
int ix = ox * stride_ + kx - padding_;
if(ix < 0 || ix >= xw) continue;
subcolumn(column_y, column_x) = xval(ib, ic, iy, ix);
}
}
}
}
}
auto bout = weight_->value().view({
output_, -1}).gemm(subcolumn).view({
output_, oh_, ow_});
auto pout = bout.ptr();
auto osptr = output_save_.ptr(ib);
if(hasbias_){
auto pbias = this->bias_->value().ptr();
for(int ot = 0; ot < output_; ++ot, ++pbias){
for(int j = 0; j < ow_ * oh_; ++j)
*osptr++ = *pout++ + *pbias;
}
}else{
memcpy(osptr, pout, sizeof(float) * output_ * oh_ * ow_);
}
}
return output_save_;
}
void Conv2dContainer::backward(const Matrix& gradient){
int xb, xc, xh, xw;
xb = x_shape_[0];
xc = x_shape_[1];
xh = x_shape_[2];
xw = x_shape_[3];
if(weight_->requires_grad()){
for(int ib = 0; ib < xb; ++ib){
auto subcolumn = column_.reference_d0(ib);
Matrix g_ib = gradient.reference_d0(ib).view({
output_, -1});
auto grad = g_ib.gemm(subcolumn, false, true).view(weight_->value().shape());
weight_->data()->backward(grad);
}
}
if(hasbias_ && bias_->requires_grad()){
Matrix bias_grad({
output_});
auto po = bias_grad.ptr();
auto pg = gradient.ptr();
size_t area = gradient.count_of(2);
for(int i = 0; i < gradient.size(0); ++i){
for(int ic = 0; ic < gradient.size(1); ++ic){
auto& bias_value = po[ic];
for(int j = 0; j < area; ++j)
bias_value += *pg++;
}
}
bias_->data()->backward(bias_grad);
}
if(x_->requires_grad()){
grad_save_.resize(x_shape_);
grad_save_.fill_zero_();
auto kcol = this->weight().value().view({
output_, -1});
for(int ib = 0; ib < xb; ++ib){
Matrix g_ib = gradient.reference_d0(ib).view({
output_, -1});
auto dcolumn = kcol.gemm(g_ib, true);
for(int ic = 0; ic < xc; ++ic){
for(int oy = 0; oy < oh_; ++oy){
for(int ox = 0; ox < ow_; ++ox){
int column_x = ox + oy * ow_;
for(int ky = 0; ky < ksize_; ++ky){
int column_idx = (ky + ic * ksize_) * ksize_;
int iy = oy * stride_ + ky - padding_;
if(iy < 0 || iy >= xh) continue;
for(int kx = 0; kx < ksize_; ++kx){
int column_y = column_idx + kx;
int ix = ox * stride_ + kx - padding_;
if(ix < 0 || ix >= xw) continue;
grad_save_(ib, ic, iy, ix) += dcolumn(column_y, column_x);
}
}
}
}
}
}
x_->backward(grad_save_);
}
}
Parameter& Conv2d::weight(){
return std::dynamic_pointer_cast<Conv2dContainer>(data_)->weight();
}
Parameter& Conv2d::bias(){
return std::dynamic_pointer_cast<Conv2dContainer>(data_)->bias();
}
Conv2d::Conv2d(int input, int output, int ksize, int stride, int padding, bool bias){
data_.reset(new Conv2dContainer(input, output, ksize, stride, padding, bias));
params_ = {
this->weight(), this->bias()};
}
Conv2d& Conv2d::operator()(const Expression& x){
std::dynamic_pointer_cast<Conv2dContainer>(data_)->assign(x);
return *this;
}
void ReLUContainer::assign(const Expression& left){
left_ = left.data();
requires_grad_ = left_->requires_grad();
}
const char* ReLUContainer::type(){
return "ReLU";
}
Matrix ReLUContainer::forward(){
save_forward_ = left_->forward().copy();
auto p = save_forward_.ptr();
for(int i = 0; i < save_forward_.numel(); ++i, ++p)
*p = std::max(0.0f, *p);
return save_forward_;
}
void ReLUContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
Matrix out = gradient.copy();
auto psave = save_forward_.ptr();
auto pout = out.ptr();
for(int i = 0; i < save_forward_.numel(); ++i){
if(psave[i] <= 0)
pout[i] = 0;
}
left_->backward(out);
}
ReLU::ReLU(){
data_.reset(new ReLUContainer());
}
ReLU& ReLU::operator()(const Expression& left){
std::dynamic_pointer_cast<ReLUContainer>(data_)->assign(left);
return *this;
}
void SigmoidContainer::assign(const Expression& left){
left_ = left.data();
requires_grad_ = left_->requires_grad();
}
Matrix SigmoidContainer::forward(){
save_forward_ = left_->forward().copy();
for(int i = 0; i < save_forward_.numel(); ++i){
auto& x = save_forward_.ptr()[i];
x = 1.0f / (1.0f + exp(-x));
}
return save_forward_;
}
void SigmoidContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
Matrix out = gradient.copy();
for(int i = 0; i < save_forward_.numel(); ++i){
auto& x = save_forward_.ptr()[i];
out.ptr()[i] *= x * (1 - x);
}
left_->backward(out);
}
const char* SigmoidContainer::type(){
return "Sigmoid";
}
Sigmoid::Sigmoid(){
data_.reset(new SigmoidContainer());
}
Sigmoid& Sigmoid::operator()(const Expression& left){
std::dynamic_pointer_cast<SigmoidContainer>(data_)->assign(left);
return *this;
}
void LogContainer::assign(const Expression& left){
left_ = left.data();
requires_grad_ = left_->requires_grad();
}
Matrix LogContainer::forward(){
save_forward_ = left_->forward().copy();
for(int i = 0; i < save_forward_.numel(); ++i){
auto& x = save_forward_.ptr()[i];
x = log(x);
}
return save_forward_;
}
void LogContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
Matrix out = gradient.copy();
for(int i = 0; i < save_forward_.numel(); ++i){
auto& x = save_forward_.ptr()[i];
out.ptr()[i] /= x;
}
left_->backward(out);
}
const char* LogContainer::type(){
return "Log";
}
Log::Log(){
data_.reset(new LogContainer());
}
Log& Log::operator()(const Expression& left){
std::dynamic_pointer_cast<LogContainer>(data_)->assign(left);
return *this;
}
void SigmoidCrossEntropyLossContainer::assign(const Expression& predict, const Expression& label){
predict_ = predict.data();
label_ = label.data();
requires_grad_ = predict_->requires_grad() || label_->requires_grad();
}
Matrix SigmoidCrossEntropyLossContainer::forward(){
auto fprob = predict_->forward().sigmoid();
auto flabel = label_->forward();
if(requires_grad_)
save_gradient_ = (fprob - flabel) * (1.0f / fprob.size(0));
float eps = 1e-5;
float sum_loss = 0;
auto pred_ptr = fprob.ptr();
auto onehot_ptr = flabel.ptr();
int numel = fprob.numel();
for(int i = 0; i < numel; ++i, ++pred_ptr, ++onehot_ptr){
auto y = *onehot_ptr;
auto p = *pred_ptr;
p = max(min(p, 1 - eps), eps);
sum_loss += -(y * log(p) + (1 - y) * log(1 - p));
}
return Matrix({
1, 1}, {
sum_loss / fprob.size(0)});
}
void SigmoidCrossEntropyLossContainer::backward(const Matrix& gradient){
if(!requires_grad_) return;
predict_->backward(save_gradient_ * gradient);
}
const char* SigmoidCrossEntropyLossContainer::type(){
return "SigmoidCrossEntropyLoss";
}
SigmoidCrossEntropyLoss::SigmoidCrossEntropyLoss(){
data_.reset(new SigmoidCrossEntropyLossContainer());
}
SigmoidCrossEntropyLoss& SigmoidCrossEntropyLoss::operator()(const Expression& predict, const Expression& label){
std::dynamic_pointer_cast<SigmoidCrossEntropyLossContainer>(data_)->assign(predict, label);
return *this;
}
Expression operator+(const Expression& a, const Expression& b){
return Add()(a, b);
}
Expression operator+(float a, const Expression& b){
return Add()(Matrix({
1, 1}, {
a}), b);
}
Expression operator+(const Expression& a, float b){
return Add()(a, Matrix({
1, 1}, {
b}));
}
Expression operator-(const Expression& a, const Expression& b){
return Sub()(a, b);
}
Expression operator-(float a, const Expression& b){
return Sub()(Matrix({
1, 1}, {
a}), b);
}
Expression operator-(const Expression& a, float b){
return Sub()(a, Matrix({
1, 1}, {
b}));
}
Expression operator*(const Expression& a, const Expression& b){
return Multiply()(a, b);
}
Expression operator*(float a, const Expression& b){
return Multiply()(Matrix({
1, 1}, {
a}), b);
}
Expression operator*(const Expression& a, float b){
return Multiply()(a, Matrix({
1, 1}, {
b}));
}
Expression Expression::view(const std::vector<int>& shape){
return View(shape)(*this);
}
Expression Expression::power(){
return Power()(*this);
}
Expression Expression::gemm(const Expression& other){
return MatMul()(*this, other);
}
Expression::Expression(const Matrix& value){
data_.reset(new MatrixContainer(value, false));
}
Matrix Expression::forward(){
auto output = data_->forward();
forward_output_shape_ = output.shape();
return output;
}
void Expression::backward(){
Matrix grad(forward_output_shape_);
grad.fill_(1);
data_->backward(grad);
}
main.cpp
#include "autodiff.hpp"
#include <vector>
#include <string>
#include <iostream>
#include <fstream>
#include <cmath>
#include <tuple>
#include <iomanip>
#include <stdarg.h>
#include <memory.h>
#include <random>
#include <algorithm>
#include <chrono>
using namespace std;
namespace Application{
static default_random_engine global_random_engine;
namespace logger{
#define INFO(...) Application::logger::__printf(__FILE__, __LINE__, __VA_ARGS__)
void __printf(const char* file, int line, const char* fmt, ...){
va_list vl;
va_start(vl, fmt);
printf("\e[32m[%s:%d]:\e[0m ", file, line);
vprintf(fmt, vl);
printf("\n");
}
};
namespace io{
struct __attribute__((packed)) mnist_labels_header_t{
unsigned int magic_number;
unsigned int number_of_items;
};
struct __attribute__((packed)) mnist_images_header_t{
unsigned int magic_number;
unsigned int number_of_images;
unsigned int number_of_rows;
unsigned int number_of_columns;
};
unsigned int inverse_byte(unsigned int v){
unsigned char* p = (unsigned char*)&v;
std::swap(p[0], p[3]);
std::swap(p[1], p[2]);
return v;
}
tuple<Matrix, Matrix> load_data(const string& image_file, const string& label_file){
Matrix images, labels;
fstream fimage(image_file, ios::binary | ios::in);
fstream flabel(label_file, ios::binary | ios::in);
mnist_images_header_t images_header;
mnist_labels_header_t labels_header;
fimage.read((char*)&images_header, sizeof(images_header));
flabel.read((char*)&labels_header, sizeof(labels_header));
images_header.number_of_images = inverse_byte(images_header.number_of_images);
labels_header.number_of_items = inverse_byte(labels_header.number_of_items);
images.resize({
(int)images_header.number_of_images, 1, 28, 28});
labels.resize({
(int)labels_header.number_of_items, 10});
std::vector<unsigned char> buffer(images.numel());
fimage.read((char*)buffer.data(), buffer.size());
for(int i = 0; i < buffer.size(); ++i)
images.ptr()[i] = (buffer[i] / 255.0f - 0.1307f) / 0.3081f;
buffer.resize(labels.size(0));
flabel.read((char*)buffer.data(), buffer.size());
for(int i = 0; i < buffer.size(); ++i)
labels.ptr(i)[buffer[i]] = 1;
return make_tuple(images, labels);
}
void print_image(const float* ptr, int rows, int cols){
for(int i = 0;i < rows * cols; ++i){
int pixel = (ptr[i] * 0.3081f + 0.1307f) * 255.0f;
if(pixel < 20)
printf("--- ");
else
printf("%03d ", pixel);
if((i + 1) % cols == 0)
printf("\n");
}
}
bool save_model(const string& file, const vector<Parameter>& params){
ofstream out(file, ios::binary | ios::out);
if(!out.is_open()){
INFO("Open %s failed.", file.c_str());
return false;
}
unsigned int header_file[] = {
0x3355FF11, params.size()};
out.write((char*)header_file, sizeof(header_file));
for(auto& p : params){
auto& m = p.value();
m.save(out);
}
return out.good();
}
bool load_model(const string& file, vector<Parameter>& params){
ifstream in(file, ios::binary | ios::in);
if(!in.is_open()){
INFO("Open %s failed.", file.c_str());
return false;
}
unsigned int header_file[2];
in.read((char*)header_file, sizeof(header_file));
if(header_file[0] != 0x3355FF11){
INFO("Invalid params file: %s", file.c_str());
return false;
}
params.resize(header_file[1]);
for(int i = 0; i < params.size(); ++i){
auto& m = params[i].value();
m.load(in);
}
return in.good();
}
};
namespace tools{
vector<int> range(int end){
vector<int> out(end);
for(int i = 0; i < end; ++i)
out[i] = i;
return out;
}
double timenow(){
return chrono::duration_cast<chrono::microseconds>(chrono::system_clock::now().time_since_epoch()).count() / 1000.0;
}
};
namespace nn{
class Model{
public:
Model():
conv1_(1, 6, 5, 2, 2),
conv2_(6, 16, 3, 2, 1),
conv3_(16, 16, 3, 1, 0),
fc1_(400, 80),
fc2_(80, 10)
{
params_ = {
conv1_.weight(), conv1_.bias(),
conv2_.weight(), conv2_.bias(),
conv3_.weight(), conv3_.bias(),
fc1_.weight(), fc1_.bias(),
fc2_.weight(), fc2_.bias()
};
}
Expression operator()(Expression x){
x = relu1_(conv1_(x));
x = relu2_(conv2_(x));
x = relu3_(conv3_(x));
x = x.view({
-1, 400});
x = relu4_(fc1_(x));
x = fc2_(x);
return x;
}
vector<Parameter>& params(){
return params_;
}
private:
Conv2d conv1_, conv2_, conv3_;
Linear fc1_, fc2_;
ReLU relu1_, relu2_, relu3_, relu4_;
vector<Parameter> params_;
};
float eval_test_accuracy(const Matrix& probability, const Matrix& labels){
int success = 0;
for(int i = 0; i < probability.size(0); ++i){
auto row_ptr = probability.ptr(i);
int predict_label = std::max_element(row_ptr, row_ptr + probability.size(1)) - row_ptr;
if(labels.ptr(i)[predict_label] == 1)
success++;
}
return success / (float)probability.size(0);
}
};
namespace optimizer{
class Optimizer{
public:
virtual void step() = 0;
virtual void zero_grad(){
for(auto& param : params_)
param.gradient().fill_(0);
}
virtual float lr() const{
return lr_;}
virtual void set_lr(float newlr){
lr_ = newlr;}
protected:
vector<Parameter> params_;
float lr_ = 0;
};
class SGDMomentum : public Optimizer{
public:
SGDMomentum(const vector<Parameter>& params, float lr, float momentum=0.9, float l2_regularization=0){
params_ = params;
lr_ = lr;
momentum_ = momentum;
delta_momentums_.resize(params_.size());
l2_regularization_ = l2_regularization;
}
void step(){
for(int i =0 ; i < params_.size(); ++i){
auto& delta_momentum = delta_momentums_[i];
auto& param = params_[i].value();
auto& grad = params_[i].gradient();
if(delta_momentum.empty())
delta_momentum.resize(param.shape());
delta_momentum = momentum_ * delta_momentum - lr_ * grad;
if(l2_regularization_ != 0)
delta_momentum -= l2_regularization_ * param;
param += delta_momentum;
}
}
private:
vector<Matrix> delta_momentums_;
float momentum_ = 0;
float l2_regularization_ = 0;
};
class AdamW : public Optimizer{
public:
AdamW(const vector<Parameter>& params, float lr=1e-3, float beta1=0.9, float beta2=0.999, float l2_regularization=0){
params_ = params;
lr_ = lr;
beta1_ = beta1;
beta2_ = beta2;
l2_regularization_ = l2_regularization;
m_.resize(params_.size());
v_.resize(params_.size());
}
void step(){
t_ ++;
for(int i = 0; i < params_.size(); ++i){
auto& m = m_[i];
auto& v = v_[i];
auto& param = params_[i].value();
auto& grad = params_[i].gradient();
if(m.empty())
m.resize(param.shape());
if(v.empty())
v.resize(param.shape());
m = beta1_ * m + (1 - beta1_) * grad;
v = beta2_ * v + (1 - beta2_) * grad.power(2.0f);
auto mt = m / (1 - std::pow(beta1_, t_));
auto vt = v / (1 - std::pow(beta2_, t_));
param -= lr_ * mt / (vt.sqrt() + eps_) + l2_regularization_ * param;
}
}
private:
vector<Matrix> m_, v_;
float t_ = 0;
float eps_ = 1e-7;
float momentum_ = 0;
float beta1_ = 0;
float beta2_ = 0;
float l2_regularization_ = 0;
};
};
int run(){
Matrix trainimages, trainlabels;
Matrix valimage, vallabels;
tie(trainimages, trainlabels) = io::load_data("mnist/train-images.idx3-ubyte", "mnist/train-labels.idx1-ubyte");
tie(valimage, vallabels) = io::load_data("mnist/t10k-images.idx3-ubyte", "mnist/t10k-labels.idx1-ubyte");
int num_images = trainimages.size(0);
int num_output = 10;
int num_epoch = 10;
float lr = 1e-2;
int batch_size = 128;
float momentum = 0.9f;
int num_batch_per_epoch = num_images / batch_size;
auto image_indexs = tools::range(num_images);
nn::Model model;
optimizer::AdamW optim(model.params(), lr, 0.9, 0.99, 1e-5);
int total_batch = 0;
auto t0 = tools::timenow();
SigmoidCrossEntropyLoss lossfn;
for(int epoch = 0; epoch < num_epoch; ++epoch){
if(epoch == 5)
optim.set_lr(optim.lr() * 0.1);
std::shuffle(image_indexs.begin(), image_indexs.end(), global_random_engine);
for(int ibatch = 0; ibatch < num_batch_per_epoch; ++ibatch, ++total_batch){
auto x = trainimages.slice(image_indexs, ibatch * batch_size, batch_size);
auto y = trainlabels.slice(image_indexs, ibatch * batch_size, batch_size);
auto loss = lossfn(model(x), y);
auto lossval = loss.forward().item();
optim.zero_grad();
loss.backward();
optim.step();
if((total_batch + 1) % 50 == 0){
auto t1 = tools::timenow();
auto batchs_time = t1 - t0;
t0 = t1;
INFO("Epoch %.2f / %d, Loss: %f, LR: %f [ %.2f ms / 50 batch ]", epoch + ibatch / (float)num_batch_per_epoch, num_epoch, lossval, optim.lr(), batchs_time);
}
}
auto test_probability = model(valimage).forward().sigmoid();
float accuracy = nn::eval_test_accuracy(test_probability, vallabels);
float test_loss = lossfn(test_probability, vallabels).forward().item();
INFO("Test Accuracy: %.2f %%, Loss: %f", accuracy * 100, test_loss);
}
INFO("Save to model.bin .");
io::save_model("model.bin", model.params());
for(int i = 0; i < valimage.size(0); ++i){
auto input = valimage.reference_d0(i).unsqueeze();
auto test_probability = model(input).forward().sigmoid();
int ilabel = test_probability.argmax(0);
float prob = test_probability(0, ilabel);
io::print_image(input.ptr(), 28, 28);
INFO("Predict %d, Confidence = %f", ilabel, prob);
printf("Prass [Enter] to next.");
getchar();
}
return 0;
}
};
int main(){
return Application::run();
}
check-grad.py
import numpy as np
import torch
import torch.nn
def dtype2int(dtype : np.dtype):
if dtype == np.float32:
return 0
elif dtype == np.float16:
return 1
elif dtype == np.float64:
return 2
elif dtype == np.int32:
return 3
elif dtype == np.uint32:
return 4
elif dtype == np.int64:
return 5
assert False, f"Unsupport dtype {dtype}"
def int2dtype(itype : int):
if itype == 0:
return np.float32
elif itype == 1:
return np.float16
elif itype == 2:
return np.float64
elif itype == 3:
return np.int32
elif itype == 4:
return np.uint32
elif itype == 5:
return np.int64
assert False, f"Unsupport itype {itype}"
def load_tensor(file):
with open(file, "rb") as f:
binary_data = f.read()
magic_number, ndims, dtype = np.frombuffer(binary_data, np.uint32, count=3, offset=0)
assert magic_number == 0xFCCFE2E2, f"{file} not a tensor file."
dims = np.frombuffer(binary_data, np.uint32, count=ndims, offset=3 * 4)
np_dtype = int2dtype(dtype)
return np.frombuffer(binary_data, np_dtype, offset=(ndims + 3) * 4).reshape(*dims)
def save_tensor(file, tensor : np.ndarray):
with open(file, "wb") as f:
f.write(np.array([0xFCCFE2E2, tensor.ndim, dtype2int(tensor.dtype)] + list(tensor.shape), dtype=np.uint32).tobytes())
f.write(tensor.tobytes())
def load_tt(file):
t = load_tensor(file)
return torch.tensor(t)
class Model(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1 = torch.nn.Conv2d(1, 5, 3, 1, 0)
self.conv1.weight.data[:] = load_tt("workspace/param0.bin")
self.conv1.bias.data[:] = load_tt("workspace/param1.bin")
self.fc1 = torch.nn.Linear(3380, 10)
self.fc1.weight.data[:] = load_tt("workspace/param2.bin").T
self.fc1.bias.data[:] = load_tt("workspace/param3.bin")
def forward(self, x):
x = torch.relu(self.conv1(x)).view(-1, 3380)
return self.fc1(x)
# 这个程序用来检验c++的求导过程是否正确,需要配合c++中的参数导出部分
model = Model()
input = load_tensor("workspace/input.bin")
output = load_tensor("workspace/output.bin")
input_p = torch.nn.parameter.Parameter(torch.tensor(input))
output_torch = model(input_p)
output_torch.sum().backward()
print(np.abs(output_torch.detach().numpy() - output).sum(), "=======outputgrad")
input_grad = load_tensor("workspace/input.grad.bin")
print(np.abs(input_p.grad.data - input_grad).sum(), "=======input")
p0_grad = load_tensor("workspace/param0.grad.bin")
print(np.abs(model.conv1.weight.grad.data - p0_grad).max(), "=======conv1.weight.grad")
p1_grad = load_tensor("workspace/param1.grad.bin")
print(np.abs(model.conv1.bias.grad.data - p1_grad).sum())
p2_grad = load_tensor("workspace/param2.grad.bin")
print(np.abs(model.fc1.weight.grad.data - p2_grad.T).sum())
p3_grad = load_tensor("workspace/param3.grad.bin")
print(np.abs(model.fc1.bias.grad.data - p3_grad).sum())
Makefile
cc := g++
name := pro
workdir := workspace
srcdir := src
objdir := objs
stdcpp := c++11
cpp_srcs := $(shell find $(srcdir) -name "*.cpp")
cpp_objs := $(cpp_srcs:.cpp=.cpp.o)
cpp_objs := $(cpp_objs:$(srcdir)/%=$(objdir)/%)
cpp_mk := $(cpp_objs:.cpp.o=.cpp.mk)
cu_srcs := $(shell find $(srcdir) -name "*.cu")
cu_objs := $(cu_srcs:.cu=.cu.o)
cu_objs := $(cu_objs:$(srcdir)/%=$(objdir)/%)
cu_mk := $(cu_objs:.cu.o=.cu.mk)
link_sys := stdc++ dl openblas
link_librarys := $(link_sys)
include_paths := src eigen3.4.90 OpenBLAS0.3.17Static/include
library_paths := OpenBLAS0.3.17Static/lib
empty :=
library_path_export := $(subst $(empty) $(empty),:,$(library_paths))
run_paths := $(foreach item,$(library_paths),-Wl,-rpath=$(item))
include_paths := $(foreach item,$(include_paths),-I$(item))
library_paths := $(foreach item,$(library_paths),-L$(item))
link_librarys := $(foreach item,$(link_librarys),-l$(item))
cpp_compile_flags := -std=$(stdcpp) -w -g -O3 -m64 -fPIC -fopenmp -pthread
link_flags := -pthread -fopenmp -Wl,-rpath='$$ORIGIN'
cpp_compile_flags += $(include_paths)
link_flags += $(library_paths) $(link_librarys) $(run_paths)
ifneq ($(MAKECMDGOALS), clean)
-include $(cpp_mk) $(cu_mk)
endif
$(name) : $(workdir)/$(name)
all : $(name)
run : $(name)
@cd $(workdir) && ./$(name) $(run_args)
$(workdir)/$(name) : $(cpp_objs) $(cu_objs)
@echo Link $@
@mkdir -p $(dir $@)
@$(cc) $^ -o $@ $(link_flags)
$(objdir)/%.cpp.o : $(srcdir)/%.cpp
@echo Compile CXX $<
@mkdir -p $(dir $@)
@$(cc) -c $< -o $@ $(cpp_compile_flags)
$(objdir)/%.cpp.mk : $(srcdir)/%.cpp
@echo Compile depends C++ $<
@mkdir -p $(dir $@)
@$(cc) -M $< -MF $@ -MT $(@:.cpp.mk=.cpp.o) $(cpp_compile_flags)
clean :
@rm -rf $(objdir) $(workdir)/$(name)
.PHONY : clean run $(name)
export LD_LIBRARY_PATH:=$(library_path_export)