NVIDIA TensorRT 8.6.11 API Reference for DRIVE OS (May 2023)

10.4 NvInfer.h

707

2341

protected:

2342

virtual ˜IScaleLayer() noexcept = default;

2343

apiv::VScaleLayer* mImpl;

2344

};

2345

2367

class ISoftMaxLayer : public ILayer

2368

{

2369

public:

2400

void setAxes(uint32 t axes) noexcept

2401

{

2402

mImpl->setAxes(axes);

2403

}

2404

2410

uint32 t getAxes() const noexcept

2411

{

2412

return mImpl->getAxes();

2413

}

2414

2415

protected:

2416

virtual ˜ISoftMaxLayer() noexcept = default;

2417

apiv::VSoftMaxLayer* mImpl;

2418

};

2419

2432

class IConcatenationLayer : public ILayer

2433

{

2434

public:

2447

void setAxis(int32 t axis) noexcept

2448

{

2449

mImpl->setAxis(axis);

2450

}

2451

2457

int32 t getAxis() const noexcept

2458

{

2459

return mImpl->getAxis();

2460

}

2461

2462

protected:

2463

virtual ˜IConcatenationLayer() noexcept = default;

2464

apiv::VConcatenationLayer* mImpl;

2465

};

2466

2474

class IDeconvolutionLayer : public ILayer

2475

{

2476

public:

2488

TRT DEPRECATED void setKernelSize(DimsHW kernelSize) noexcept

2489

{

2490

mImpl->setKernelSize(kernelSize);

2491

}

2492

2500

TRT DEPRECATED DimsHW getKernelSize() const noexcept

2501

{

2502

return mImpl->getKernelSize();

2503

}

2504

2512

void setNbOutputMaps(int32 t nbOutputMaps) noexcept

2513

{

2514

mImpl->setNbOutputMaps(nbOutputMaps);

2515

}

2516

2522

int32 t getNbOutputMaps() const noexcept

2523

{

2524

return mImpl->getNbOutputMaps();

2525

}

2526

2538

TRT DEPRECATED void setStride(DimsHW stride) noexcept

2539

{

2540

mImpl->setStride(stride);

2541

}

2542

2550

TRT DEPRECATED DimsHW getStride() const noexcept

2551

{

2552

return mImpl->getStride();

2553

}

2554

2570

TRT DEPRECATED void setPadding(DimsHW padding) noexcept

2571

{

2572

mImpl->setPadding(padding);

2573

}

2574

2584

TRT DEPRECATED DimsHW getPadding() const noexcept

2585

{

708

File Documentation

2586

return mImpl->getPadding();

2587

}

2588

2604

void setNbGroups(int32 t nbGroups) noexcept

2605

{

2606

mImpl->setNbGroups(nbGroups);

2607

}

2608

2614

int32 t getNbGroups() const noexcept

2615

{

2616

return mImpl->getNbGroups();

2617

}

2618

2628

void setKernelWeights(Weights weights) noexcept

2629

{

2630

mImpl->setKernelWeights(weights);

2631

}

2632

2638

Weights getKernelWeights() const noexcept

2639

{

2640

return mImpl->getKernelWeights();

2641

}

2642

2653

void setBiasWeights(Weights weights) noexcept

2654

{

2655

mImpl->setBiasWeights(weights);

2656

}

2657

2663

Weights getBiasWeights() const noexcept

2664

{

2665

return mImpl->getBiasWeights();

2666

}

2667

2681

void setPrePadding(Dims padding) noexcept

2682

{

2683

mImpl->setPrePadding(padding);

2684

}

2685

2691

Dims getPrePadding() const noexcept

2692

{

2693

return mImpl->getPrePadding();

2694

}

2695

2709

void setPostPadding(Dims padding) noexcept

2710

{

2711

mImpl->setPostPadding(padding);

2712

}

2713

2719

Dims getPostPadding() const noexcept

2720

{

2721

return mImpl->getPostPadding();

2722

}

2723

2733

void setPaddingMode(PaddingMode paddingMode) noexcept

2734

{

2735

mImpl->setPaddingMode(paddingMode);

2736

}

2737

2745

PaddingMode getPaddingMode() const noexcept

2746

{

2747

return mImpl->getPaddingMode();

2748

}

2749

2760

void setKernelSizeNd(Dims kernelSize) noexcept

2761

{

2762

mImpl->setKernelSizeNd(kernelSize);

2763

}

2764

2770

Dims getKernelSizeNd() const noexcept

2771

{

2772

return mImpl->getKernelSizeNd();

2773

}

2774

2787

void setStrideNd(Dims stride) noexcept

2788

{

2789

mImpl->setStrideNd(stride);

2790

}

2791

2797

Dims getStrideNd() const noexcept

2798

{

2799

return mImpl->getStrideNd();

10.4 NvInfer.h

709

2800

}

2801

2815

void setPaddingNd(Dims padding) noexcept

2816

{

2817

mImpl->setPaddingNd(padding);

2818

}

2819

2827

Dims getPaddingNd() const noexcept

2828

{

2829

return mImpl->getPaddingNd();

2830

}

2831

2843

using ILayer::setInput;

2844

2851

void setDilationNd(Dims dilation) noexcept

2852

{

2853

mImpl->setDilationNd(dilation);

2854

}

2855

2861

Dims getDilationNd() const noexcept

2862

{

2863

return mImpl->getDilationNd();

2864

}

2865

2866

protected:

2867

virtual ˜IDeconvolutionLayer() noexcept = default;

2868

apiv::VDeconvolutionLayer* mImpl;

2869

};

2870

2884

enum class ElementWiseOperation : int32 t

2885

{

2886

kSUM = 0,

2887

kPROD = 1,

2888

kMAX = 2,

2889

kMIN = 3,

2890

kSUB = 4,

2891

kDIV = 5,

2892

kPOW = 6,

2893

kFLOOR DIV = 7,

2894

kAND = 8,

2895

kOR = 9,

2896

kXOR = 10,

2897

kEQUAL = 11,

2898

kGREATER = 12,

2899

kLESS = 13

2900

};

2901

2902

namespace impl

2903

{

2909

template <>

2910

struct EnumMaxImpl<ElementWiseOperation>

2911

{

2912

static constexpr int32 t kVALUE = 14;

2913

};

2914

} // namespace impl

2915

2935

class IElementWiseLayer : public ILayer

2936

{

2937

public:

2947

void setOperation(ElementWiseOperation op) noexcept

2948

{

2949

return mImpl->setOperation(op);

2950

}

2951

2959

ElementWiseOperation getOperation() const noexcept

2960

{

2961

return mImpl->getOperation();

2962

}

2963

2964

protected:

2965

apiv::VElementWiseLayer* mImpl;

2966

virtual ˜IElementWiseLayer() noexcept = default;

2967

};

2968

2974

enum class GatherMode : int32 t

2975

{

2976

kDEFAULT = 0,

2977

kELEMENT = 1,

2978

kND = 2

2979

};

2980

710

File Documentation

2986

template <>

2987

constexpr inline int32 t EnumMax<GatherMode>() noexcept

2988

{

2989

return 3;

2990

}

2991

3070

class IGatherLayer : public ILayer

3071

{

3072

public:

3082

void setGatherAxis(int32 t axis) noexcept

3083

{

3084

mImpl->setGatherAxis(axis);

3085

}

3086

3093

int32 t getGatherAxis() const noexcept

3094

{

3095

return mImpl->getGatherAxis();

3096

}

3097

3114

void setNbElementWiseDims(int32 t elementWiseDims) noexcept

3115

{

3116

mImpl->setNbElementWiseDims(elementWiseDims);

3117

}

3118

3124

int32 t getNbElementWiseDims() const noexcept

3125

{

3126

return mImpl->getNbElementWiseDims();

3127

}

3128

3134

void setMode(GatherMode mode) noexcept

3135

{

3136

mImpl->setMode(mode);

3137

}

3138

3144

GatherMode getMode() const noexcept

3145

{

3146

return mImpl->getMode();

3147

}

3148

3149

protected:

3150

apiv::VGatherLayer* mImpl;

3151

virtual ˜IGatherLayer() noexcept = default;

3152

};

3153

3233

enum class RNNOperation : int32 t

3234

{

3235

kRELU = 0,

3236

kTANH = 1,

3237

kLSTM = 2,

3238

kGRU = 3

3239

};

3240

3246

template <>

3247

constexpr inline int32 t EnumMax<RNNOperation>() noexcept

3248

{

3249

return 4;

3250

}

3251

3259

enum class RNNDirection : int32 t

3260

{

3261

kUNIDIRECTION = 0,

3262

kBIDIRECTION = 1

3263

};

3264

3270

template <>

3271

constexpr inline int32 t EnumMax<RNNDirection>() noexcept

3272

{

3273

return 2;

3274

}

3275

3291

enum class RNNInputMode : int32 t

3292

{

3293

kLINEAR = 0,

3294

kSKIP = 1

3295

};

3296

3302

template <>

3303

constexpr inline int32 t EnumMax<RNNInputMode>() noexcept

3304

{

3305

return 2;

3306

}

10.4 NvInfer.h

711

3307

3315

enum class RNNGateType : int32 t

3316

{

3317

kINPUT = 0,

3318

kOUTPUT = 1,

3319

kFORGET = 2,

3320

kUPDATE = 3,

3321

kRESET = 4,

3322

kCELL = 5,

3323

kHIDDEN = 6

3324

};

3325

3331

template <>

3332

constexpr inline int32 t EnumMax<RNNGateType>() noexcept

3333

{

3334

return 7;

3335

}

3336

3349

class TRT DEPRECATED IRNNv2Layer : public ILayer

3350

{

3351

public:

3352

int32 t getLayerCount() const noexcept

3353

{

3354

return mImpl->getLayerCount();

3355

}

3356

int32 t getHiddenSize() const noexcept

3357

{

3358

return mImpl->getHiddenSize();

3359

}

3360

int32 t getMaxSeqLength() const noexcept

3361

{

3362

return mImpl->getMaxSeqLength();

3363

}

3364

int32 t getDataLength() const noexcept

3365

{

3366

return mImpl->getDataLength();

3367

}

3368

3383

void setSequenceLengths(ITensor& seqLengths) noexcept

3384

{

3385

return mImpl->setSequenceLengths(seqLengths);

3386

}

3387

3395

ITensor_* getSequenceLengths() const noexcept

3396

{

3397

return mImpl->getSequenceLengths();

3398

}

3399

3405

void setOperation(RNNOperation op) noexcept

3406

{

3407

mImpl->setOperation(op);

3408

}

3409

3415

RNNOperation getOperation() const noexcept

3416

{

3417

return mImpl->getOperation();

3418

}

3419

3425

void setInputMode(RNNInputMode op) noexcept

3426

{

3427

mImpl->setInputMode(op);

3428

}

3429

3435

RNNInputMode getInputMode() const noexcept

3436

{

3437

return mImpl->getInputMode();

3438

}

3439

3450

void setDirection(RNNDirection op) noexcept

3451

{

3452

mImpl->setDirection(op);

3453

}

3454

3460

RNNDirection getDirection() const noexcept

3461

{

3462

return mImpl->getDirection();

3463

}

3464

3518

void setWeightsForGate(int32 t layerIndex, RNNGateType gate, bool isW, Weights weights) noexcept

3519

{

3520

mImpl->setWeightsForGate(layerIndex, gate, isW, weights);

712

File Documentation

3521

}

3522

3528

Weights getWeightsForGate(int32 t layerIndex, RNNGateType gate, bool isW) const noexcept

3529

{

3530

return mImpl->getWeightsForGate(layerIndex, gate, isW);

3531

}

3532

3552

void setBiasForGate(int32 t layerIndex, RNNGateType gate, bool isW, Weights bias) noexcept

3553

{

3554

mImpl->setBiasForGate(layerIndex, gate, isW, bias);

3555

}

3556

3562

Weights getBiasForGate(int32 t layerIndex, RNNGateType gate, bool isW) const noexcept

3563

{

3564

return mImpl->getBiasForGate(layerIndex, gate, isW);

3565

}

3566

3580

void setHiddenState(ITensor& hidden) noexcept

3581

{

3582

mImpl->setHiddenState(hidden);

3583

}

3584

3590

ITensor_* getHiddenState() const noexcept

3591

{

3592

return mImpl->getHiddenState();

3593

}

3594

3610

void setCellState(ITensor& cell) noexcept

3611

{

3612

mImpl->setCellState(cell);

3613

}

3614

3620

ITensor_* getCellState() const noexcept

3621

{

3622

return mImpl->getCellState();

3623

}

3624

3625

protected:

3626

apiv::VRNNv2Layer* mImpl;

3627

virtual ˜IRNNv2Layer() noexcept = default;

3628

};

3629

3639

class IPluginV2Layer : public ILayer

3640

{

3641

public:

3647

IPluginV2& getPlugin() noexcept

3648

{

3649

return mImpl->getPlugin();

3650

}

3651

3652

protected:

3653

apiv::VPluginV2Layer* mImpl;

3654

virtual ˜IPluginV2Layer() noexcept = default;

3655

};

3656

3674

enum class UnaryOperation : int32 t

3675

{

3676

kEXP = 0,

3677

kLOG = 1,

3678

kSQRT = 2,

3679

kRECIP = 3,

3680

kABS = 4,

3681

kNEG = 5,

3682

kSIN = 6,

3683

kCOS = 7,

3684

kTAN = 8,

3685

kSINH = 9,

3686

kCOSH = 10,

3687

kASIN = 11,

3688

kACOS = 12,

3689

kATAN = 13,

3690

kASINH = 14,

3691

kACOSH = 15,

3692

kATANH = 16,

3693

kCEIL = 17,

3694

kFLOOR = 18,

3695

kERF = 19,

3696

kNOT = 20,

3697

kSIGN = 21,

3698

kROUND = 22,

3699

kISINF = 23,

10.4 NvInfer.h

713

3700

};

3701

3707

template <>

3708

constexpr inline int32 t EnumMax<UnaryOperation>() noexcept

3709

{

3710

return 24;

3711

}

3712

3720

class IUnaryLayer : public ILayer

3721

{

3722

public:

3730

void setOperation(UnaryOperation op) noexcept

3731

{

3732

mImpl->setOperation(op);

3733

}

3734

3740

UnaryOperation getOperation() const noexcept

3741

{

3742

return mImpl->getOperation();

3743

}

3744

3745

protected:

3746

apiv::VUnaryLayer* mImpl;

3747

virtual ˜IUnaryLayer() noexcept = default;

3748

};

3749

3768

enum class ReduceOperation : int32 t

3769

{

3770

kSUM = 0,

3771

kPROD = 1,

3772

kMAX = 2,

3773

kMIN = 3,

3774

kAVG = 4

3775

};

3776

3782

template <>

3783

constexpr inline int32 t EnumMax<ReduceOperation>() noexcept

3784

{

3785

return 5;

3786

}

3787

3795

class IReduceLayer : public ILayer

3796

{

3797

public:

3803

void setOperation(ReduceOperation op) noexcept

3804

{

3805

mImpl->setOperation(op);

3806

}

3807

3813

ReduceOperation getOperation() const noexcept

3814

{

3815

return mImpl->getOperation();

3816

}

3817

3823

void setReduceAxes(uint32 t reduceAxes) noexcept

3824

{

3825

mImpl->setReduceAxes(reduceAxes);

3826

}

3827

3833

uint32 t getReduceAxes() const noexcept

3834

{

3835

return mImpl->getReduceAxes();

3836

}

3837

3843

void setKeepDimensions(bool keepDimensions) noexcept

3844

{

3845

mImpl->setKeepDimensions(keepDimensions);

3846

}

3847

3853

bool getKeepDimensions() const noexcept

3854

{

3855

return mImpl->getKeepDimensions();

3856

}

3857

3858

protected:

3859

apiv::VReduceLayer* mImpl;

3860

virtual ˜IReduceLayer() noexcept = default;

3861

};

3862

3873

class IPaddingLayer : public ILayer

3874

{

714

File Documentation

3875

public:

3885

TRT DEPRECATED void setPrePadding(DimsHW padding) noexcept

3886

{

3887

mImpl->setPrePadding(padding);

3888

}

3889

3897

TRT DEPRECATED DimsHW getPrePadding() const noexcept

3898

{

3899

return mImpl->getPrePadding();

3900

}

3901

3911

TRT DEPRECATED void setPostPadding(DimsHW padding) noexcept

3912

{

3913

mImpl->setPostPadding(padding);

3914

}

3915

3923

TRT DEPRECATED DimsHW getPostPadding() const noexcept

3924

{

3925

return mImpl->getPostPadding();

3926

}

3927

3937

void setPrePaddingNd(Dims padding) noexcept

3938

{

3939

mImpl->setPrePaddingNd(padding);

3940

}

3941

3949

Dims getPrePaddingNd() const noexcept

3950

{

3951

return mImpl->getPrePaddingNd();

3952

}

3953

3963

void setPostPaddingNd(Dims padding) noexcept

3964

{

3965

mImpl->setPostPaddingNd(padding);

3966

}

3967

3975

Dims getPostPaddingNd() const noexcept

3976

{

3977

return mImpl->getPostPaddingNd();

3978

}

3979

3980

protected:

3981

apiv::VPaddingLayer* mImpl;

3982

virtual ˜IPaddingLayer() noexcept = default;

3983

};

3984

3985

struct Permutation

3986

{

3993

int32 t order[Dims::MAX DIMS];

3994

};

3995

4008

class IShuffleLayer : public ILayer

4009

{

4010

public:

4020

void setFirstTranspose(Permutation permutation) noexcept

4021

{

4022

mImpl->setFirstTranspose(permutation);

4023

}

4024

4032

Permutation getFirstTranspose() const noexcept

4033

{

4034

return mImpl->getFirstTranspose();

4035

}

4036

4057

void setReshapeDimensions(Dims dimensions) noexcept

4058

{

4059

mImpl->setReshapeDimensions(dimensions);

4060

}

4061

4070

Dims getReshapeDimensions() const noexcept

4071

{

4072

return mImpl->getReshapeDimensions();

4073

}

4074

4080

4103

using ILayer::setInput;

4104

4117

void setSecondTranspose(Permutation permutation) noexcept

4118

{

4119

mImpl->setSecondTranspose(permutation);

4120

}

10.4 NvInfer.h

715

4121

4129

Permutation getSecondTranspose() const noexcept

4130

{

4131

return mImpl->getSecondTranspose();

4132

}

4133

4145

void setZeroIsPlaceholder(bool zeroIsPlaceholder) noexcept

4146

{

4147

return mImpl->setZeroIsPlaceholder(zeroIsPlaceholder);

4148

}

4149

4158

bool getZeroIsPlaceholder() const noexcept

4159

{

4160

return mImpl->getZeroIsPlaceholder();

4161

}

4162

4163

protected:

4164

apiv::VShuffleLayer* mImpl;

4165

virtual ˜IShuffleLayer() noexcept = default;

4166

};

4167

4173

enum class SampleMode : int32 t

4174

{

4175

kSTRICT BOUNDS = 0,

4176

kDEFAULT TRT DEPRECATED ENUM = kSTRICT BOUNDS,

4177

kWRAP = 1,

4178

kCLAMP = 2,

4179

kFILL = 3,

4180

kREFLECT = 4,

4183

};

4184

4186

using SliceMode = SampleMode;

4187

4193

template <>

4194

constexpr inline int32 t EnumMax<SampleMode>() noexcept

4195

{

4196

return 5;

4197

}

4198

4241

class ISliceLayer : public ILayer

4242

{

4243

public:

4253

void setStart(Dims start) noexcept

4254

{

4255

mImpl->setStart(start);

4256

}

4257

4268

Dims getStart() const noexcept

4269

{

4270

return mImpl->getStart();

4271

}

4272

4282

void setSize(Dims size) noexcept

4283

{

4284

return mImpl->setSize(size);

4285

}

4286

4297

Dims getSize() const noexcept

4298

{

4299

return mImpl->getSize();

4300

}

4301

4311

void setStride(Dims stride) noexcept

4312

{

4313

mImpl->setStride(stride);

4314

}

4315

4326

Dims getStride() const noexcept

4327

{

4328

return mImpl->getStride();

4329

}

4330

4336

void setMode(SliceMode mode) noexcept

4337

{

4338

mImpl->setMode(mode);

4339

}

4340

4346

SliceMode getMode() const noexcept

4347

{

4348

return mImpl->getMode();

4349

}

716

File Documentation

4350

4374

using ILayer::setInput;

4375

4376

protected:

4377

apiv::VSliceLayer* mImpl;

4378

virtual ˜ISliceLayer() noexcept = default;

4379

};

4380

4393

class IShapeLayer : public ILayer

4394

{

4395

protected:

4396

apiv::VShapeLayer* mImpl;

4397

virtual ˜IShapeLayer() noexcept = default;

4398

};

4399

4405

enum class TopKOperation : int32 t

4406

{

4407

kMAX = 0,

4408

kMIN = 1,

4409

};

4410

4416

template <>

4417

constexpr inline int32 t EnumMax<TopKOperation>() noexcept

4418

{

4419

return 2;

4420

}

4421

4433

class ITopKLayer : public ILayer

4434

{

4435

public:

4441

void setOperation(TopKOperation op) noexcept

4442

{

4443

mImpl->setOperation(op);

4444

}

4445

4451

TopKOperation getOperation() const noexcept

4452

{

4453

return mImpl->getOperation();

4454

}

4455

4465

void setK(int32 t k) noexcept

4466

{

4467

mImpl->setK(k);

4468

}

4469

4479

int32 t getK() const noexcept

4480

{

4481

return mImpl->getK();

4482

}

4483

4489

void setReduceAxes(uint32 t reduceAxes) noexcept

4490

{

4491

mImpl->setReduceAxes(reduceAxes);

4492

}

4493

4499

uint32 t getReduceAxes() const noexcept

4500

{

4501

return mImpl->getReduceAxes();

4502

}

4503

4518

using ILayer::setInput;

4519

4520

protected:

4521

apiv::VTopKLayer* mImpl;

4522

virtual ˜ITopKLayer() noexcept = default;

4523

};

4524

4531

enum class MatrixOperation : int32 t

4532

{

4536

kNONE,

4537

4539

kTRANSPOSE,

4540

4551

kVECTOR

4552

};

4553

4559

template <>

4560

constexpr inline int32 t EnumMax<MatrixOperation>() noexcept

4561

{

4562

return 3;

4563

}

10.4 NvInfer.h

717

4564

4590

class IMatrixMultiplyLayer : public ILayer

4591

{

4592

public:

4599

void setOperation(int32 t index, MatrixOperation op) noexcept

4600

{

4601

mImpl->setOperation(index, op);

4602

}

4603

4611

MatrixOperation getOperation(int32 t index) const noexcept

4612

{

4613

return mImpl->getOperation(index);

4614

}

4615

4616

protected:

4617

apiv::VMatrixMultiplyLayer* mImpl;

4618

virtual ˜IMatrixMultiplyLayer() noexcept = default;

4619

};

4620

4642

class INonZeroLayer : public ILayer

4643

{

4644

protected:

4645

virtual ˜INonZeroLayer() noexcept = default;

4646

apiv::VNonZeroLayer* mImpl;

4647

};

4648

4663

class IRaggedSoftMaxLayer : public ILayer

4664

{

4665

protected:

4666

apiv::VRaggedSoftMaxLayer* mImpl;

4667

virtual ˜IRaggedSoftMaxLayer() noexcept = default;

4668

};

4669

4697

class IIdentityLayer : public ILayer

4698

{

4699

protected:

4700

apiv::VIdentityLayer* mImpl;

4701

virtual ˜IIdentityLayer() noexcept = default;

4702

};

4703

4710

class ICastLayer : public ILayer

4711

{

4712

public:

4716

void setToType(DataType toType) noexcept

4717

{

4718

mImpl->setToType(toType);

4719

}

4720

4724

DataType getToType() const noexcept

4725

{

4726

return mImpl->getToType();

4727

}

4728

4729

protected:

4730

apiv::VCastLayer* mImpl;

4731

virtual ˜ICastLayer() noexcept = default;

4732

};

4733

4742

class IConstantLayer : public ILayer

4743

{

4744

public:

4754

void setWeights(Weights weights) noexcept

4755

{

4756

mImpl->setWeights(weights);

4757

}

4758

4764

Weights getWeights() const noexcept

4765

{

4766

return mImpl->getWeights();

4767

}

4768

4776

void setDimensions(Dims dimensions) noexcept

4777

{

4778

mImpl->setDimensions(dimensions);

4779

}

4780

4788

Dims getDimensions() const noexcept

4789

{

4790

return mImpl->getDimensions();

4791

}

4792

718

File Documentation

4793

protected:

4794

apiv::VConstantLayer* mImpl;

4795

virtual ˜IConstantLayer() noexcept = default;

4796

};

4797

4807

class IParametricReLULayer : public ILayer

4808

{

4809

protected:

4810

apiv::VParametricReLULayer* mImpl;

4811

virtual ˜IParametricReLULayer() noexcept = default;

4812

};

4813

4819

enum class InterpolationMode : int32 t

4820

{

4821

kNEAREST = 0,

4822

kLINEAR = 1,

4823

kCUBIC = 2

4824

};

4825

4827

using ResizeMode = InterpolationMode;

4828

4829

namespace impl

4830

{

4836

template <>

4837

struct EnumMaxImpl<InterpolationMode>

4838

{

4839

static constexpr int32 t kVALUE = 3;

4840

};

4841

} // namespace impl

4842

4850

enum class ResizeCoordinateTransformation : int32 t

4851

{

4864

kALIGN CORNERS = 0,

4865

4872

kASYMMETRIC = 1,

4873

4880

kHALF PIXEL = 2,

4881

};

4882

4883

namespace impl

4884

{

4890

template <>

4891

struct EnumMaxImpl<ResizeCoordinateTransformation>

4892

{

4893

static constexpr int32 t kVALUE = 3;

4894

};

4895

} // namespace impl

4896

4904

enum class ResizeSelector : int32 t

4905

{

4907

kFORMULA = 0,

4908

4910

kUPPER = 1,

4911

};

4912

4913

namespace impl

4914

{

4920

template <>

4921

struct EnumMaxImpl<ResizeSelector>

4922

{

4923

static constexpr int32 t kVALUE = 2;

4924

};

4925

} // namespace impl

4926

4934

enum class ResizeRoundMode : int32 t

4935

{

4937

kHALF UP = 0,

4938

4940

kHALF DOWN = 1,

4941

4943

kFLOOR = 2,

4944

4946

kCEIL = 3,

4947

};

4948

4949

namespace impl

4950

{

4956

template <>

4957

struct EnumMaxImpl<ResizeRoundMode>

4958

{

4959

static constexpr int32 t kVALUE = 4;

10.4 NvInfer.h

719

4960

};

4961

} // namespace impl

4962

4999

class IResizeLayer : public ILayer

5000

{

5001

public:

5020

void setOutputDimensions(Dims dimensions) noexcept

5021

{

5022

return mImpl->setOutputDimensions(dimensions);

5023

}

5024

5030

Dims getOutputDimensions() const noexcept

5031

{

5032

return mImpl->getOutputDimensions();

5033

}

5034

5060

void setScales(float const* scales, int32 t nbScales) noexcept

5061

{

5062

mImpl->setScales(scales, nbScales);

5063

}

5064

5079

int32 t getScales(int32 t size, float* scales) const noexcept

5080

{

5081

return mImpl->getScales(size, scales);

5082

}

5083

5091

void setResizeMode(ResizeMode resizeMode) noexcept

5092

{

5093

mImpl->setResizeMode(resizeMode);

5094

}

5095

5101

ResizeMode getResizeMode() const noexcept

5102

{

5103

return mImpl->getResizeMode();

5104

}

5105

5117

TRT DEPRECATED void setAlignCorners(bool alignCorners) noexcept

5118

{

5119

mImpl->setAlignCorners(alignCorners);

5120

}

5121

5129

TRT DEPRECATED bool getAlignCorners() const noexcept

5130

{

5131

return mImpl->getAlignCorners();

5132

}

5133

5153

using ILayer::setInput;

5154

5164

void setCoordinateTransformation(ResizeCoordinateTransformation coordTransform) noexcept

5165

{

5166

mImpl->setCoordinateTransformation(coordTransform);

5167

}

5168

5174

ResizeCoordinateTransformation getCoordinateTransformation() const noexcept

5175

{

5176

return mImpl->getCoordinateTransformation();

5177

}

5178

5189

void setSelectorForSinglePixel(ResizeSelector selector) noexcept

5190

{

5191

mImpl->setSelectorForSinglePixel(selector);

5192

}

5193

5199

ResizeSelector getSelectorForSinglePixel() const noexcept

5200

{

5201

return mImpl->getSelectorForSinglePixel();

5202

}

5203

5213

void setNearestRounding(ResizeRoundMode value) noexcept

5214

{

5215

mImpl->setNearestRounding(value);

5216

}

5217

5223

ResizeRoundMode getNearestRounding() const noexcept

5224

{

5225

return mImpl->getNearestRounding();

5226

}

5227

5245

void setCubicCoeff(float A) noexcept

5246

{

5247

mImpl->setCubicCoeff(A);

720

File Documentation

5248

}

5249

5255

float getCubicCoeff() const noexcept

5256

{

5257

return mImpl->getCubicCoeff();

5258

}

5259

5268

void setExcludeOutside(bool excludeFlag) noexcept

5269

{

5270

mImpl->setExcludeOutside(excludeFlag);

5271

}

5272

5278

bool getExcludeOutside() const noexcept

5279

{

5280

return mImpl->getExcludeOutside();

5281

}

5282

5283

protected:

5284

virtual ˜IResizeLayer() noexcept = default;

5285

apiv::VResizeLayer* mImpl;

5286

};

5287

5289

enum class LoopOutput : int32 t

5290

{

5292

kLAST VALUE = 0,

5293

5295

kCONCATENATE = 1,

5296

5298

kREVERSE = 2

5299

};

5300

5306

template <>

5307

constexpr inline int32 t EnumMax<LoopOutput>() noexcept

5308

{

5309

return 3;

5310

}

5311

5313

enum class TripLimit : int32 t

5314

{

5315

5316

kCOUNT = 0,

5317

kWHILE = 1

5318

};

5319

5325

template <>

5326

constexpr inline int32 t EnumMax<TripLimit>() noexcept

5327

{

5328

return 2;

5329

}

5330

5331

class ILoop;

5332

5333

class ILoopBoundaryLayer : public ILayer

5334

{

5335

public:

5337

ILoop_* getLoop() const noexcept

5338

{

5339

return mBoundary->getLoop();

5340

}

5341

5342

protected:

5343

virtual ˜ILoopBoundaryLayer() noexcept = default;

5344

apiv::VLoopBoundaryLayer* mBoundary;

5345

};

5346

5352

class IIfConditionalBoundaryLayer : public ILayer

5353

{

5354

public:

5356

IIfConditional_* getConditional() const noexcept

5357

{

5358

return mBoundary->getConditional();

5359

}

5360

5361

protected:

5362

virtual ˜IIfConditionalBoundaryLayer() noexcept = default;

5363

apiv::VConditionalBoundaryLayer* mBoundary;

5364

};

5365

5369

class IConditionLayer : public IIfConditionalBoundaryLayer

5370

{

5371

public:

10.4 NvInfer.h

721

5372

protected:

5373

virtual ˜IConditionLayer() noexcept = default;

5374

apiv::VConditionLayer* mImpl;

5375

};

5376

5382

class IIfConditionalOutputLayer : public IIfConditionalBoundaryLayer

5383

{

5384

public:

5385

protected:

5386

virtual ˜IIfConditionalOutputLayer() noexcept = default;

5387

apiv::VConditionalOutputLayer* mImpl;

5388

};

5389

5393

class IIfConditionalInputLayer : public IIfConditionalBoundaryLayer

5394

{

5395

public:

5396

protected:

5397

virtual ˜IIfConditionalInputLayer() noexcept = default;

5398

apiv::VConditionalInputLayer* mImpl;

5399

};

5400

5422

class IIfConditional : public INoCopy

5423

{

5424

public:

5434

IConditionLayer_* setCondition(ITensor& condition) noexcept

5435

{

5436

return mImpl->setCondition(condition);

5437

}

5438

5450

IIfConditionalOutputLayer_* addOutput(ITensor& trueSubgraphOutput, ITensor& falseSubgraphOutput)

noexcept

5451

{

5452

return mImpl->addOutput(trueSubgraphOutput, falseSubgraphOutput);

5453

}

5454

5462

IIfConditionalInputLayer_* addInput(ITensor& input) noexcept

5463

{

5464

return mImpl->addInput(input);

5465

}

5466

5477

void setName(char const* name) noexcept

5478

{

5479

mImpl->setName(name);

5480

}

5481

5487

char const_* getName() const noexcept

5488

{

5489

return mImpl->getName();

5490

}

5491

5492

protected:

5493

virtual ˜IIfConditional() noexcept = default;

5494

apiv::VIfConditional* mImpl;

5495

};

5496

5497

5498

class IRecurrenceLayer : public ILoopBoundaryLayer

5499

{

5500

public:

5506

5519

using ILayer::setInput;

5520

5521

protected:

5522

virtual ˜IRecurrenceLayer() noexcept = default;

5523

apiv::VRecurrenceLayer* mImpl;

5524

};

5525

5543

class ILoopOutputLayer : public ILoopBoundaryLayer

5544

{

5545

public:

5546

LoopOutput getLoopOutput() const noexcept

5547

{

5548

return mImpl->getLoopOutput();

5549

}

5550

5563

void setAxis(int32 t axis) noexcept

5564

{

5565

mImpl->setAxis(axis);

5566

}

5567

5569

int32 t getAxis() const noexcept

722

File Documentation

5570

{

5571

return mImpl->getAxis();

5572

}

5573

5579

5594

using ILayer::setInput;

5595

5596

protected:

5597

virtual ˜ILoopOutputLayer() noexcept = default;

5598

apiv::VLoopOutputLayer* mImpl;

5599

};

5600

5601

class ITripLimitLayer : public ILoopBoundaryLayer

5602

{

5603

public:

5604

TripLimit getTripLimit() const noexcept

5605

{

5606

return mImpl->getTripLimit();

5607

}

5608

5609

protected:

5610

virtual ˜ITripLimitLayer() noexcept = default;

5611

apiv::VTripLimitLayer* mImpl;

5612

};

5613

5614

class IIteratorLayer : public ILoopBoundaryLayer

5615

{

5616

public:

5618

void setAxis(int32 t axis) noexcept

5619

{

5620

mImpl->setAxis(axis);

5621

}

5622

5624

int32 t getAxis() const noexcept

5625

{

5626

return mImpl->getAxis();

5627

}

5628

5634

void setReverse(bool reverse) noexcept

5635

{

5636

mImpl->setReverse(reverse);

5637

}

5638

5640

bool getReverse() const noexcept

5641

{

5642

return mImpl->getReverse();

5643

}

5644

5645

protected:

5646

virtual ˜IIteratorLayer() noexcept = default;

5647

apiv::VIteratorLayer* mImpl;

5648

};

5649

5655

class ILoop : public INoCopy

5656

{

5657

public:

5664

IRecurrenceLayer_* addRecurrence(ITensor& initialValue) noexcept

5665

{

5666

return mImpl->addRecurrence(initialValue);

5667

}

5668

5685

ITripLimitLayer_* addTripLimit(ITensor& tensor, TripLimit limit) noexcept

5686

{

5687

return mImpl->addTripLimit(tensor, limit);

5688

}

5689

5698

IIteratorLayer_* addIterator(ITensor& tensor, int32 t axis = 0, bool reverse = false) noexcept

5699

{

5700

return mImpl->addIterator(tensor, axis, reverse);

5701

}

5702

5710

ILoopOutputLayer_* addLoopOutput(ITensor& tensor, LoopOutput outputKind, int32 t axis = 0) noexcept

5711

{

5712

return mImpl->addLoopOutput(tensor, outputKind, axis);

5713

}

5714

5725

void setName(char const* name) noexcept

5726

{

5727

mImpl->setName(name);

5728

}

5729

10.4 NvInfer.h

723

5735

char const_* getName() const noexcept

5736

{

5737

return mImpl->getName();

5738

}

5739

5740

protected:

5741

virtual ˜ILoop() noexcept = default;

5742

apiv::VLoop* mImpl;

5743

};

5744

5748

class ISelectLayer : public ILayer

5749

{

5750

protected:

5751

virtual ˜ISelectLayer() noexcept = default;

5752

apiv::VSelectLayer* mImpl;

5753

};

5754

5769

class IAssertionLayer : public ILayer

5770

{

5771

public:

5780

void setMessage(char const* message) noexcept

5781

{

5782

mImpl->setMessage(message);

5783

}

5784

5790

char const_* getMessage() const noexcept

5791

{

5792

return mImpl->getMessage();

5793

}

5794

5795

protected:

5796

virtual ˜IAssertionLayer() noexcept = default;

5797

5798

apiv::VAssertionLayer* mImpl;

5799

};

5800

5808

enum class FillOperation : int32 t

5809

{

5810

kLINSPACE = 0,

5811

kRANDOM UNIFORM = 1,

5812

kRANDOM NORMAL = 2

5813

};

5814

5820

template <>

5821

constexpr inline int32 t EnumMax<FillOperation>() noexcept

5822

{

5823

return 3;

5824

}

5825

5851

class IFillLayer : public ILayer

5852

{

5853

public:

5862

5863

void setDimensions(Dims dimensions) noexcept

5864

{

5865

mImpl->setDimensions(dimensions);

5866

}

5867

5878

Dims getDimensions() const noexcept

5879

{

5880

return mImpl->getDimensions();

5881

}

5882

5888

void setOperation(FillOperation op) noexcept

5889

{

5890

mImpl->setOperation(op);

5891

}

5892

5898

FillOperation getOperation() const noexcept

5899

{

5900

return mImpl->getOperation();

5901

}

5902

5916

5917

void setAlpha(double alpha) noexcept

5918

{

5919

mImpl->setAlpha(alpha);

5920

}

5921

5932

double getAlpha() const noexcept

5933

{

724

File Documentation

5934

return mImpl->getAlpha();

5935

}

5936

5951

void setBeta(double beta) noexcept

5952

{

5953

mImpl->setBeta(beta);

5954

}

5955

5966

double getBeta() const noexcept

5967

{

5968

return mImpl->getBeta();

5969

}

5970

6003

using ILayer::setInput;

6004

6005

protected:

6006

virtual ˜IFillLayer() noexcept = default;

6007

apiv::VFillLayer* mImpl;

6008

};

6009

6067

class IQuantizeLayer : public ILayer

6068

{

6069

public:

6078

int32 t getAxis() const noexcept

6079

{

6080

return mImpl->getAxis();

6081

}

6089

void setAxis(int32 t axis) noexcept

6090

{

6091

mImpl->setAxis(axis);

6092

}

6093

6094

protected:

6095

virtual ˜IQuantizeLayer() noexcept = default;

6096

apiv::VQuantizeLayer* mImpl;

6097

};

6098

6153

class IDequantizeLayer : public ILayer

6154

{

6155

public:

6164

int32 t getAxis() const noexcept

6165

{

6166

return mImpl->getAxis();

6167

}

6175

void setAxis(int32 t axis) noexcept

6176

{

6177

mImpl->setAxis(axis);

6178

}

6179

6180

protected:

6181

virtual ˜IDequantizeLayer() noexcept = default;

6182

apiv::VDequantizeLayer* mImpl;

6183

};

6184

6220

class IEinsumLayer : public ILayer

6221

{

6222

public:

6232

bool setEquation(char const* equation) noexcept

6233

{

6234

return mImpl->setEquation(equation);

6235

}

6236

6242

char const_* getEquation() const noexcept

6243

{

6244

return mImpl->getEquation();

6245

}

6246

6247

protected:

6248

virtual ˜IEinsumLayer() noexcept = default;

6249

apiv::VEinsumLayer* mImpl;

6250

};

6251

6257

enum class ScatterMode : int32 t

6258

{

6259

kELEMENT = 0,

6260

kND = 1,

6261

};

6262

6268

template <>

6269

constexpr inline int32 t EnumMax<ScatterMode>() noexcept

6270

{

10.4 NvInfer.h

725

6271

return 2;

6272

}

6273

6330

class IScatterLayer : public ILayer

6331

{

6332

public:

6338

void setMode(ScatterMode mode) noexcept

6339

{

6340

mImpl->setMode(mode);

6341

}

6342

6348

ScatterMode getMode() const noexcept

6349

{

6350

return mImpl->getMode();

6351

}

6352

6358

void setAxis(int32 t axis) noexcept

6359

{

6360

mImpl->setAxis(axis);

6361

}

6362

6366

int32 t getAxis() const noexcept

6367

{

6368

return mImpl->getAxis();

6369

}

6370

6371

protected:

6372

apiv::VScatterLayer* mImpl;

6373

virtual ˜IScatterLayer() noexcept = default;

6374

}; // class IScatterLayer

6375

6403

class IOneHotLayer : public ILayer

6404

{

6405

public:

6411

void setAxis(int32 t axis) noexcept

6412

{

6413

mImpl->setAxis(axis);

6414

}

6415

6419

int32 t getAxis() const noexcept

6420

{

6421

return mImpl->getAxis();

6422

}

6423

6424

protected:

6425

apiv::VOneHotLayer* mImpl;

6426

};

6427

6438

class IGridSampleLayer : public ILayer

6439

{

6440

public:

6446

void setInterpolationMode(InterpolationMode mode) noexcept

6447

{

6448

mImpl->setInterpolationMode(mode);

6449

}

6450

6458

InterpolationMode getInterpolationMode() const noexcept

6459

{

6460

return mImpl->getInterpolationMode();

6461

}

6462

6468

void setAlignCorners(bool alignCorners) noexcept

6469

{

6470

mImpl->setAlignCorners(alignCorners);

6471

}

6472

6480

bool getAlignCorners() const noexcept

6481

{

6482

return mImpl->getAlignCorners();

6483

}

6484

6492

bool setSampleMode(SampleMode mode) noexcept

6493

{

6494

return mImpl->setSampleMode(mode);

6495

}

6496

6504

SampleMode getSampleMode() const noexcept

6505

{

6506

return mImpl->getSampleMode();

6507

}

6508

726

File Documentation

6509

protected:

6510

apiv::VGridSampleLayer* mImpl;

6511

virtual ˜IGridSampleLayer() noexcept = default;

6512

}; // class IGridSampleLayer

6513

6519

enum class BoundingBoxFormat : int32 t

6520

{

6522

kCORNER PAIRS = 0,

6524

kCENTER SIZES = 1

6525

};

6526

6532

template <>

6533

constexpr inline int32 t EnumMax<BoundingBoxFormat>() noexcept

6534

{

6535

return 2;

6536

}

6537

6581

class INMSLayer : public ILayer

6582

{

6583

public:

6593

void setBoundingBoxFormat(BoundingBoxFormat fmt) noexcept

6594

{

6595

mImpl->setBoundingBoxFormat(fmt);

6596

}

6597

6605

BoundingBoxFormat getBoundingBoxFormat() const noexcept

6606

{

6607

return mImpl->getBoundingBoxFormat();

6608

}

6609

6619

void setTopKBoxLimit(int32 t limit) noexcept

6620

{

6621

mImpl->setTopKBoxLimit(limit);

6622

}

6623

6629

int32 t getTopKBoxLimit() const noexcept

6630

{

6631

return mImpl->getTopKBoxLimit();

6632

}

6633

6652

using ILayer::setInput;

6653

6654

protected:

6655

apiv::VNMSLayer* mImpl;

6656

virtual ˜INMSLayer() noexcept = default;

6657

}; // class INMSLayer

6658

6670

class IReverseSequenceLayer: public ILayer

6671

{

6672

public:

6681

void setBatchAxis(int32 t batchAxis) noexcept

6682

{

6683

mImpl->setBatchAxis(batchAxis);

6684

}

6685

6691

int32 t getBatchAxis() const noexcept

6692

{

6693

return mImpl->getBatchAxis();

6694

}

6695

6704

void setSequenceAxis(int32 t sequenceAxis) noexcept

6705

{

6706

mImpl->setSequenceAxis(sequenceAxis);

6707

}

6708

6714

int32 t getSequenceAxis() const noexcept

6715

{

6716

return mImpl->getSequenceAxis();

6717

}

6718

6719

protected:

6720

apiv::VReverseSequenceLayer* mImpl;

6721

virtual ˜IReverseSequenceLayer() noexcept = default;

6722

}; // class IReverseSequenceLayer

6723

6740

6741

class INormalizationLayer : public ILayer

6742

{

6743

public:

6750

void setEpsilon(float eps) noexcept

6751

{

10.4 NvInfer.h

727

6752

return mImpl->setEpsilon(eps);

6753

}

6754

6759

float getEpsilon() const noexcept

6760

{

6761

return mImpl->getEpsilon();

6762

}

6763

6768

void setAxes(uint32 t axesMask) noexcept

6769

{

6770

return mImpl->setAxes(axesMask);

6771

}

6772

6777

uint32 t getAxes() const noexcept

6778

{

6779

return mImpl->getAxes();

6780

}

6781

6797

void setNbGroups(int32 t nbGroups) noexcept

6798

{

6799

return mImpl->setNbGroups(nbGroups);

6800

}

6801

6806

int32 t getNbGroups() const noexcept

6807

{

6808

return mImpl->getNbGroups();

6809

}

6810

6824

void setComputePrecision(DataType type) noexcept

6825

{

6826

return mImpl->setComputePrecision(type);

6827

}

6828

6833

DataType getComputePrecision() const noexcept

6834

{

6835

return mImpl->getComputePrecision();

6836

}

6837

6838

protected:

6839

apiv::VNormalizationLayer* mImpl;

6840

virtual ˜INormalizationLayer() noexcept = default;

6841 };

6842

6862

class INetworkDefinition : public INoCopy

6863 {

6864

public:

6865

virtual ˜INetworkDefinition() noexcept = default;

6866

6904

ITensor* addInput(char const* name, DataType type, Dims dimensions) noexcept

6905

{

6906

return mImpl->addInput(name, type, dimensions);

6907

}

6908

6918

void markOutput(ITensor& tensor) noexcept

6919

{

6920

mImpl->markOutput(tensor);

6921

}

6922

6941

TRT DEPRECATED IConvolutionLayer_* addConvolution(

6942

ITensor& input, int32 t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights

biasWeights) noexcept

6943

{

6944

return mImpl->addConvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);

6945

}

6946

6964

TRT DEPRECATED IFullyConnectedLayer_* addFullyConnected(

6965

ITensor& input, int32 t nbOutputs, Weights kernelWeights, Weights biasWeights) noexcept

6966

{

6967

return mImpl->addFullyConnected(input, nbOutputs, kernelWeights, biasWeights);

6968

}

6969

6984

IActivationLayer_* addActivation(ITensor& input, ActivationType type) noexcept

6985

{

6986

return mImpl->addActivation(input, type);

6987

}

6988

7003

TRT DEPRECATED IPoolingLayer_* addPooling(ITensor& input, PoolingType type, DimsHW windowSize) noexcept

7004

{

7005

return mImpl->addPooling(input, type, windowSize);

7006

}

7007

728

File Documentation

7022

ILRNLayer_* addLRN(ITensor& input, int32 t window, float alpha, float beta, float k) noexcept

7023

{

7024

return mImpl->addLRN(input, window, alpha, beta, k);

7025

}

7026

7049

IScaleLayer_* addScale(ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power)

noexcept

7050

{

7051

return mImpl->addScale(input, mode, shift, scale, power);

7052

}

7053

7062

ISoftMaxLayer_* addSoftMax(ITensor& input) noexcept

7063

{

7064

return mImpl->addSoftMax(input);

7065

}

7066

7079

IConcatenationLayer_* addConcatenation(ITensor_* const* inputs, int32 t nbInputs) noexcept

7080

{

7081

return mImpl->addConcatenation(inputs, nbInputs);

7082

}

7083

7102

TRT DEPRECATED IDeconvolutionLayer_* addDeconvolution(

7103

ITensor& input, int32 t nbOutputMaps, DimsHW kernelSize, Weights kernelWeights, Weights

biasWeights) noexcept

7104

{

7105

return mImpl->addDeconvolution(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);

7106

}

7107

7130

IElementWiseLayer_* addElementWise(ITensor& input1, ITensor& input2, ElementWiseOperation op) noexcept

7131

{

7132

return mImpl->addElementWise(input1, input2, op);

7133

}

7134

7152

IUnaryLayer_* addUnary(ITensor& input, UnaryOperation operation) noexcept

7153

{

7154

return mImpl->addUnary(input, operation);

7155

}

7156

7169

TRT DEPRECATED IPaddingLayer_* addPadding(ITensor& input, DimsHW prePadding, DimsHW postPadding)

noexcept

7170

{

7171

return mImpl->addPadding(input, prePadding, postPadding);

7172

}

7173

7183

IShuffleLayer_* addShuffle(ITensor& input) noexcept

7184

{

7185

return mImpl->addShuffle(input);

7186

}

7187

7200

IOneHotLayer_* addOneHot(ITensor& indices, ITensor& values, ITensor& depth, int32 t axis) noexcept

7201

{

7202

return mImpl->addOneHot(indices, values, depth, axis);

7203

}

7204

7212

int32 t getNbLayers() const noexcept

7213

{

7214

return mImpl->getNbLayers();

7215

}

7216

7226

ILayer_* getLayer(int32 t index) const noexcept

7227

{

7228

return mImpl->getLayer(index);

7229

}

7230

7238

int32 t getNbInputs() const noexcept

7239

{

7240

return mImpl->getNbInputs();

7241

}

7242

7254

ITensor_* getInput(int32 t index) const noexcept

7255

{

7256

return mImpl->getInput(index);

7257

}

7258

7268

int32 t getNbOutputs() const noexcept

7269

{

7270

return mImpl->getNbOutputs();

7271

}

7272

7284

ITensor_* getOutput(int32 t index) const noexcept

7285

{

10.4 NvInfer.h

729

7286

return mImpl->getOutput(index);

7287

}

7288

7296

TRT DEPRECATED void destroy() noexcept

7297

{

7298

delete this;

7299

}

7300

7323

IReduceLayer_* addReduce(

7324

ITensor& input, ReduceOperation operation, uint32 t reduceAxes, bool keepDimensions) noexcept

7325

{

7326

return mImpl->addReduce(input, operation, reduceAxes, keepDimensions);

7327

}

7328

7358

ITopKLayer_* addTopK(ITensor& input, TopKOperation op, int32 t k, uint32 t reduceAxes) noexcept

7359

{

7360

return mImpl->addTopK(input, op, k, reduceAxes);

7361

}

7362

7374

IGatherLayer_* addGather(ITensor& data, ITensor& indices, int32 t axis) noexcept

7375

{

7376

return mImpl->addGather(data, indices, axis);

7377

}

7378

7390

IGatherLayer_* addGatherV2(ITensor& data, ITensor& indices, GatherMode mode) noexcept

7391

{

7392

return mImpl->addGatherV2(data, indices, mode);

7393

}

7394

7408

IRaggedSoftMaxLayer_* addRaggedSoftMax(ITensor& input, ITensor& bounds) noexcept

7409

{

7410

return mImpl->addRaggedSoftMax(input, bounds);

7411

}

7412

7429

IMatrixMultiplyLayer_* addMatrixMultiply(

7430

ITensor& input0, MatrixOperation op0, ITensor& input1, MatrixOperation op1) noexcept

7431

{

7432

return mImpl->addMatrixMultiply(input0, op0, input1, op1);

7433

}

7434

7444

INonZeroLayer_* addNonZero(ITensor& input) noexcept

7445

{

7446

return mImpl->addNonZero(input);

7447

}

7448

7471

IConstantLayer_* addConstant(Dims dimensions, Weights weights) noexcept

7472

{

7473

return mImpl->addConstant(dimensions, weights);

7474

}

7475

7541

TRT DEPRECATED IRNNv2Layer_* addRNNv2(

7542

ITensor& input, int32 t layerCount, int32 t hiddenSize, int32 t maxSeqLen, RNNOperation op) noexcept

7543

{

7544

return mImpl->addRNNv2(input, layerCount, hiddenSize, maxSeqLen, op);

7545

}

7546

7556

IIdentityLayer_* addIdentity(ITensor& input) noexcept

7557

{

7558

return mImpl->addIdentity(input);

7559

}

7560

7571

ICastLayer_* addCast(ITensor& input, DataType toType) noexcept

7572

{

7573

return mImpl->addCast(input, toType);

7574

}

7575

7586

void removeTensor(ITensor& tensor) noexcept

7587

{

7588

mImpl->removeTensor(tensor);

7589

}

7590

7598

void unmarkOutput(ITensor& tensor) noexcept

7599

{

7600

mImpl->unmarkOutput(tensor);

7601

}

7602

7617

IPluginV2Layer_* addPluginV2(ITensor_* const* inputs, int32 t nbInputs, IPluginV2& plugin) noexcept

7618

{

7619

return mImpl->addPluginV2(inputs, nbInputs, plugin);

7620

}

7621

730

File Documentation

7636

ISliceLayer_* addSlice(ITensor& input, Dims start, Dims size, Dims stride) noexcept

7637

{

7638

return mImpl->addSlice(input, start, size, stride);

7639

}

7640

7660

void setName(char const* name) noexcept

7661

{

7662

mImpl->setName(name);

7663

}

7664

7674

char const_* getName() const noexcept

7675

{

7676

return mImpl->getName();

7677

}

7678

7690

IShapeLayer_* addShape(ITensor& input) noexcept

7691

{

7692

return mImpl->addShape(input);

7693

}

7694

7708

bool hasImplicitBatchDimension() const noexcept

7709

{

7710

return mImpl->hasImplicitBatchDimension();

7711

}

7712

7726

bool markOutputForShapes(ITensor& tensor) noexcept

7727

{

7728

return mImpl->markOutputForShapes(tensor);

7729

}

7730

7738

bool unmarkOutputForShapes(ITensor& tensor) noexcept

7739

{

7740

return mImpl->unmarkOutputForShapes(tensor);

7741

}

7742

7756

IParametricReLULayer_* addParametricReLU(ITensor& input, ITensor& slope) noexcept

7757

{

7758

return mImpl->addParametricReLU(input, slope);

7759

}

7760

7778

IConvolutionLayer_* addConvolutionNd(

7779

ITensor& input, int32 t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights)

noexcept

7780

{

7781

return mImpl->addConvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);

7782

}

7783

7798

IPoolingLayer_* addPoolingNd(ITensor& input, PoolingType type, Dims windowSize) noexcept

7799

{

7800

return mImpl->addPoolingNd(input, type, windowSize);

7801

}

7802

7817

7820

IDeconvolutionLayer_* addDeconvolutionNd(

7821

ITensor& input, int32 t nbOutputMaps, Dims kernelSize, Weights kernelWeights, Weights biasWeights)

noexcept

7822

{

7823

return mImpl->addDeconvolutionNd(input, nbOutputMaps, kernelSize, kernelWeights, biasWeights);

7824

}

7825

7857

IScaleLayer_* addScaleNd(

7858

ITensor& input, ScaleMode mode, Weights shift, Weights scale, Weights power, int32 t channelAxis)

noexcept

7859

{

7860

return mImpl->addScaleNd(input, mode, shift, scale, power, channelAxis);

7861

}

7862

7873

IResizeLayer_* addResize(ITensor& input) noexcept

7874

{

7875

return mImpl->addResize(input);

7876

}

7877

7887

TRT DEPRECATED bool hasExplicitPrecision() const noexcept

7888

{

7889

return mImpl->hasExplicitPrecision();

7890

}

7891

7903

ILoop_* addLoop() noexcept

7904

{

7905

return mImpl->addLoop();

7906

}

10.4 NvInfer.h

731

7907

7943

ISelectLayer_* addSelect(ITensor& condition, ITensor& thenInput, ITensor& elseInput) noexcept

7944

{

7945

return mImpl->addSelect(condition, thenInput, elseInput);

7946

}

7947

7960

IAssertionLayer_* addAssertion(ITensor& condition, char const* message) noexcept

7961

{

7962

return mImpl->addAssertion(condition, message);

7963

}

7964

7983

IFillLayer_* addFill(Dims dimensions, FillOperation op) noexcept

7984

{

7985

return mImpl->addFill(dimensions, op);

7986

}

7987

8000

TRT DEPRECATED IPaddingLayer_* addPaddingNd(ITensor& input, Dims prePadding, Dims postPadding) noexcept

8001

{

8002

return mImpl->addPaddingNd(input, prePadding, postPadding);

8003

}

8004

8023

bool setWeightsName(Weights weights, char const* name) noexcept

8024

{

8025

return mImpl->setWeightsName(weights, name);

8026

}

8027

8039

8042

void setErrorRecorder(IErrorRecorder* recorder) noexcept

8043

{

8044

mImpl->setErrorRecorder(recorder);

8045

}

8046

8057

IErrorRecorder_* getErrorRecorder() const noexcept

8058

{

8059

return mImpl->getErrorRecorder();

8060

}

8061

8076

IDequantizeLayer_* addDequantize(ITensor& input, ITensor& scale) noexcept

8077

{

8078

return mImpl->addDequantize(input, scale);

8079

}

8080

8096

IScatterLayer_* addScatter(ITensor& data, ITensor& indices, ITensor& updates, ScatterMode mode) noexcept

8097

{

8098

return mImpl->addScatter(data, indices, updates, mode);

8099

}

8100

8115

IQuantizeLayer_* addQuantize(ITensor& input, ITensor& scale) noexcept

8116

{

8117

return mImpl->addQuantize(input, scale);

8118

}

8119

8130

IIfConditional_* addIfConditional() noexcept

8131

{

8132

return mImpl->addIfConditional();

8133

}

8134

8144

IEinsumLayer_* addEinsum(ITensor_* const* inputs, int32 t nbInputs, char const* equation) noexcept

8145

{

8146

return mImpl->addEinsum(inputs, nbInputs, equation);

8147

}

8148

8160

IGridSampleLayer_* addGridSample(ITensor& input, ITensor& grid) noexcept

8161

{

8162

return mImpl->addGridSample(input, grid);

8163

}

8164

8178

INMSLayer_* addNMS(ITensor& boxes, ITensor& scores, ITensor& maxOutputBoxesPerClass) noexcept

8179

{

8180

return mImpl->addNMS(boxes, scores, maxOutputBoxesPerClass);

8181

}

8182

8195

IReverseSequenceLayer_* addReverseSequence(ITensor& input, ITensor& sequenceLens) noexcept

8196

{

8197

return mImpl->addReverseSequence(input, sequenceLens);

8198

}

8199

8221

INormalizationLayer_* addNormalization(

8222

ITensor& input, ITensor& scale, ITensor& bias, uint32 t axesMask) noexcept

8223

{

8224

return mImpl->addNormalization(input, scale, bias, axesMask);

732

File Documentation

8225

}

8226

8233

virtual IBuilder& getBuilder() const noexcept

8234

{

8235

return mImpl->getBuilder();

8236

}

8237

8238

protected:

8239

apiv::VNetworkDefinition* mImpl;

8240

};

8241

8247

enum class CalibrationAlgoType : int32 t

8248

{

8249

kLEGACY CALIBRATION = 0,

8250

kENTROPY CALIBRATION = 1,

8251

kENTROPY CALIBRATION 2 = 2,

8252

kMINMAX CALIBRATION = 3,

8253

};

8254

8260

template <>

8261

constexpr inline int32 t EnumMax<CalibrationAlgoType>() noexcept

8262

{

8263

return 4;

8264

}

8265

8277

class IInt8Calibrator

8278

{

8279

public:

8285

virtual int32 t getBatchSize() const noexcept = 0;

8286

8300

virtual bool getBatch(void* bindings[], char const* names[], int32 t nbBindings) noexcept = 0;

8301

8316

virtual void const* readCalibrationCache(std::size t& length) noexcept = 0;

8317

8326

virtual void writeCalibrationCache(void const* ptr, std::size t length) noexcept = 0;

8327

8333

virtual CalibrationAlgoType getAlgorithm() noexcept = 0;

8334

8335

virtual ˜IInt8Calibrator() noexcept = default;

8336

};

8337

8342

class IInt8EntropyCalibrator : public IInt8Calibrator

8343

{

8344

public:

8348

CalibrationAlgoType getAlgorithm() noexcept override

8349

{

8350

return CalibrationAlgoType::kENTROPY CALIBRATION;

8351

}

8352

8353

virtual ˜IInt8EntropyCalibrator() noexcept = default;

8354

};

8355

8360

class IInt8EntropyCalibrator2 : public IInt8Calibrator

8361

{

8362

public:

8366

CalibrationAlgoType getAlgorithm() noexcept override

8367

{

8368

return CalibrationAlgoType::kENTROPY CALIBRATION 2;

8369

}

8370

8371

virtual ˜IInt8EntropyCalibrator2() noexcept = default;

8372

};

8373

8377

class IInt8MinMaxCalibrator : public IInt8Calibrator

8378

{

8379

public:

8383

CalibrationAlgoType getAlgorithm() noexcept override

8384

{

8385

return CalibrationAlgoType::kMINMAX CALIBRATION;

8386

}

8387

8388

virtual ˜IInt8MinMaxCalibrator() noexcept = default;

8389

};

8390

8395

class IInt8LegacyCalibrator : public IInt8Calibrator

8396

{

8397

public:

8401

CalibrationAlgoType getAlgorithm() noexcept override

8402

{

8403

return CalibrationAlgoType::kLEGACY CALIBRATION;

8404

}

10.4 NvInfer.h

733

8405

8412

virtual double getQuantile() const noexcept = 0;

8413

8420

virtual double getRegressionCutoff() const noexcept = 0;

8421

8434

virtual void const* readHistogramCache(std::size t& length) noexcept = 0;

8435

8444

virtual void writeHistogramCache(void const* ptr, std::size t length) noexcept = 0;

8445

8446

virtual ˜IInt8LegacyCalibrator() noexcept = default;

8447

};

8448

8459

class IAlgorithmIOInfo : public INoCopy

8460

{

8461

public:

8470

TRT DEPRECATED TensorFormat getTensorFormat() const noexcept

8471

{

8472

return mImpl->getTensorFormat();

8473

}

8474

8480

DataType getDataType() const noexcept

8481

{

8482

return mImpl->getDataType();

8483

}

8484

8491

Dims getStrides() const noexcept

8492

{

8493

return mImpl->getStrides();

8494

}

8495

8501

int64 t getVectorizedDim() const noexcept

8502

{

8503

return mImpl->getVectorizedDim();

8504

}

8505

8512

int64 t getComponentsPerElement() const noexcept

8513

{

8514

return mImpl->getComponentsPerElement();

8515

}

8516

8517

protected:

8518

virtual ˜IAlgorithmIOInfo() noexcept = default;

8519

apiv::VAlgorithmIOInfo* mImpl;

8520

};

8521

8533

class IAlgorithmVariant : public INoCopy

8534

{

8535

public:

8539

int64 t getImplementation() const noexcept

8540

{

8541

return mImpl->getImplementation();

8542

}

8543

8547

int64 t getTactic() const noexcept

8548

{

8549

return mImpl->getTactic();

8550

}

8551

8552

protected:

8553

virtual ˜IAlgorithmVariant() noexcept = default;

8554

apiv::VAlgorithmVariant* mImpl;

8555

};

8556

8565

class IAlgorithmContext : public INoCopy

8566

{

8567

public:

8572

char const_* getName() const noexcept

8573

{

8574

return mImpl->getName();

8575

}

8576

8583

Dims getDimensions(int32 t index, OptProfileSelector select) const noexcept

8584

{

8585

return mImpl->getDimensions(index, select);

8586

}

8587

8591

int32 t getNbInputs() const noexcept

8592

{

8593

return mImpl->getNbInputs();

8594

}

8595

734

File Documentation

8599

int32 t getNbOutputs() const noexcept

8600

{

8601

return mImpl->getNbOutputs();

8602

}

8603

8604

protected:

8605

virtual ˜IAlgorithmContext() noexcept = default;

8606

apiv::VAlgorithmContext* mImpl;

8607

};

8608

8618

class IAlgorithm : public INoCopy

8619

{

8620

public:

8631

TRT DEPRECATED IAlgorithmIOInfo const& getAlgorithmIOInfo(int32 t index) const noexcept

8632

{

8633

return mImpl->getAlgorithmIOInfo(index);

8634

}

8635

8639

IAlgorithmVariant const& getAlgorithmVariant() const noexcept

8640

{

8641

return mImpl->getAlgorithmVariant();

8642

}

8643

8647

float getTimingMSec() const noexcept

8648

{

8649

return mImpl->getTimingMSec();

8650

}

8651

8655

std::size t getWorkspaceSize() const noexcept

8656

{

8657

return mImpl->getWorkspaceSize();

8658

}

8659

8668

IAlgorithmIOInfo const_* getAlgorithmIOInfoByIndex(int32 t index) const noexcept

8669

{

8670

return mImpl->getAlgorithmIOInfoByIndex(index);

8671

}

8672

8673

protected:

8674

virtual ˜IAlgorithm() noexcept = default;

8675

apiv::VAlgorithm* mImpl;

8676

}; // IAlgorithm

8677

8686

class IAlgorithmSelector

8687

{

8688

public:

8703

virtual int32 t selectAlgorithms(IAlgorithmContext const& context, IAlgorithm const_* const* choices,

8704

int32 t nbChoices, int32 t* selection) noexcept = 0;

8715

virtual void reportAlgorithms(IAlgorithmContext const_* const* algoContexts, IAlgorithm const_* const_*

algoChoices,

8716

int32 t nbAlgorithms) noexcept = 0;

8717

8718

virtual ˜IAlgorithmSelector() noexcept = default;

8719

};

8720

8727

using QuantizationFlags = uint32 t;

8728

8736

enum class QuantizationFlag : int32 t

8737

{

8741

kCALIBRATE BEFORE FUSION = 0

8742

};

8743

8749

template <>

8750

constexpr inline int32 t EnumMax<QuantizationFlag>() noexcept

8751

{

8752

return 1;

8753

}

8754

8761

using BuilderFlags = uint32 t;

8762

8770

enum class BuilderFlag : int32 t

8771

{

8772

kFP16 = 0,

8773

kINT8 = 1,

8774

kDEBUG = 2,

8775

kGPU FALLBACK = 3,

8776

8788

kSTRICT TYPES TRT DEPRECATED ENUM = 4,

8789

8790

kREFIT = 5,

8791

kDISABLE TIMING CACHE = 6,

10.4 NvInfer.h

735

8792

8796

kTF32 = 7,

8797

8799

kSPARSE WEIGHTS = 8,

8800

8807

kSAFETY SCOPE = 9,

8808

8810

kOBEY PRECISION CONSTRAINTS = 10,

8811

8814

kPREFER PRECISION CONSTRAINTS = 11,

8815

8819

kDIRECT IO = 12,

8820

8822

kREJECT EMPTY ALGORITHMS = 13,

8823

8829

kENABLE TACTIC HEURISTIC = 14,

8830

8839

kVERSION COMPATIBLE = 15,

8840

8848

kEXCLUDE LEAN RUNTIME = 16,

8849

8852

kFP8 = 17

8853

};

8854

8860

template <>

8861

constexpr inline int32 t EnumMax<BuilderFlag>() noexcept

8862

{

8863

return 18;

8864

}

8865

8876

class ITimingCache : public INoCopy

8877

{

8878

public:

8879

virtual ˜ITimingCache() noexcept = default;

8880

8890

nvinfer1::IHostMemory* serialize() const noexcept

8891

{

8892

return mImpl->serialize();

8893

}

8894

8914

bool combine(ITimingCache const& inputCache, bool ignoreMismatch) noexcept

8915

{

8916

return mImpl->combine(inputCache, ignoreMismatch);

8917

}

8918

8924

bool reset() noexcept

8925

{

8926

return mImpl->reset();

8927

}

8928

8929

protected:

8930

apiv::VTimingCache* mImpl;

8931

};

8932

8940

enum class MemoryPoolType : int32 t

8941

{

8949

kWORKSPACE = 0,

8950

8957

kDLA MANAGED SRAM = 1,

8958

8964

kDLA LOCAL DRAM = 2,

8965

8971

kDLA GLOBAL DRAM = 3,

8972

8980

kTACTIC DRAM = 4,

8981

};

8982

8988

template <>

8989

constexpr inline int32 t EnumMax<MemoryPoolType>() noexcept

8990

{

8991

return 5;

8992

}

8993

9002

enum class PreviewFeature : int32 t

9003

{

9014

kFASTER DYNAMIC SHAPES 0805 TRT DEPRECATED ENUM = 0,

9015

9030

kDISABLE EXTERNAL TACTIC SOURCES FOR CORE 0805 = 1,

9031

9037

kPROFILE SHARING 0806 = 2,

9038

};

736

File Documentation

9039

namespace impl

9040

{

9046

template <>

9047

struct EnumMaxImpl<PreviewFeature>

9048

{

9049

static constexpr int32 t kVALUE = 3;

9050

};

9051

} // namespace impl

9052

9056

enum class HardwareCompatibilityLevel : int32 t

9057

{

9060

kNONE = 0,

9061

9066

kAMPERE PLUS = 1,

9067

};

9068

9069

namespace impl

9070

{

9076

template <>

9077

struct EnumMaxImpl<HardwareCompatibilityLevel>

9078

{

9079

static constexpr int32 t kVALUE = 2;

9080

};

9081

} // namespace impl

9082

9090

class IBuilderConfig : public INoCopy

9091

{

9092

public:

9093

virtual ˜IBuilderConfig() noexcept = default;

9094

9107

TRT DEPRECATED virtual void setMinTimingIterations(int32 t minTiming) noexcept

9108

{

9109

mImpl->setMinTimingIterations(minTiming);

9110

}

9111

9121

TRT DEPRECATED virtual int32 t getMinTimingIterations() const noexcept

9122

{

9123

return mImpl->getMinTimingIterations();

9124

}

9125

9134

virtual void setAvgTimingIterations(int32 t avgTiming) noexcept

9135

{

9136

mImpl->setAvgTimingIterations(avgTiming);

9137

}

9138

9146

int32 t getAvgTimingIterations() const noexcept

9147

{

9148

return mImpl->getAvgTimingIterations();

9149

}

9150

9159

void setEngineCapability(EngineCapability capability) noexcept

9160

{

9161

mImpl->setEngineCapability(capability);

9162

}

9163

9171

EngineCapability getEngineCapability() const noexcept

9172

{

9173

return mImpl->getEngineCapability();

9174

}

9175

9181

void setInt8Calibrator(IInt8Calibrator* calibrator) noexcept

9182

{

9183

mImpl->setInt8Calibrator(calibrator);

9184

}

9185

9189

IInt8Calibrator_* getInt8Calibrator() const noexcept

9190

{

9191

return mImpl->getInt8Calibrator();

9192

}

9193

9204

TRT DEPRECATED void setMaxWorkspaceSize(std::size t workspaceSize) noexcept

9205

{

9206

mImpl->setMaxWorkspaceSize(workspaceSize);

9207

}

9208

9221

TRT DEPRECATED std::size t getMaxWorkspaceSize() const noexcept

9222

{

9223

return mImpl->getMaxWorkspaceSize();

9224

}

9225

9238

void setFlags(BuilderFlags builderFlags) noexcept

10.4 NvInfer.h

737

9239

{

9240

mImpl->setFlags(builderFlags);

9241

}

9242

9250

BuilderFlags getFlags() const noexcept

9251

{

9252

return mImpl->getFlags();

9253

}

9254

9262

void clearFlag(BuilderFlag builderFlag) noexcept

9263

{

9264

mImpl->clearFlag(builderFlag);

9265

}

9266

9274

void setFlag(BuilderFlag builderFlag) noexcept

9275

{

9276

mImpl->setFlag(builderFlag);

9277

}

9278

9286

bool getFlag(BuilderFlag builderFlag) const noexcept

9287

{

9288

return mImpl->getFlag(builderFlag);

9289

}

9290

9302

void setDeviceType(ILayer const* layer, DeviceType deviceType) noexcept

9303

{

9304

mImpl->setDeviceType(layer, deviceType);

9305

}

9306

9311

DeviceType getDeviceType(ILayer const* layer) const noexcept

9312

{

9313

return mImpl->getDeviceType(layer);

9314

}

9315

9321

bool isDeviceTypeSet(ILayer const* layer) const noexcept

9322

{

9323

return mImpl->isDeviceTypeSet(layer);

9324

}

9325

9331

void resetDeviceType(ILayer const* layer) noexcept

9332

{

9333

mImpl->resetDeviceType(layer);

9334

}

9335

9340

bool canRunOnDLA(ILayer const* layer) const noexcept

9341

{

9342

return mImpl->canRunOnDLA(layer);

9343

}

9344

9355

void setDLACore(int32 t dlaCore) noexcept

9356

{

9357

mImpl->setDLACore(dlaCore);

9358

}

9359

9364

int32 t getDLACore() const noexcept

9365

{

9366

return mImpl->getDLACore();

9367

}

9368

9374

void setDefaultDeviceType(DeviceType deviceType) noexcept

9375

{

9376

mImpl->setDefaultDeviceType(deviceType);

9377

}

9378

9384

DeviceType getDefaultDeviceType() const noexcept

9385

{

9386

return mImpl->getDefaultDeviceType();

9387

}

9388

9394

void reset() noexcept

9395

{

9396

mImpl->reset();

9397

}

9398

9408

TRT DEPRECATED void destroy() noexcept

9409

{

9410

delete this;

9411

}

9412

9420

void setProfileStream(const cudaStream t stream) noexcept

9421

{

738

File Documentation

9422

return mImpl->setProfileStream(stream);

9423

}

9424

9432

cudaStream t getProfileStream() const noexcept

9433

{

9434

return mImpl->getProfileStream();

9435

}

9436

9448

int32 t addOptimizationProfile(IOptimizationProfile const* profile) noexcept

9449

{

9450

return mImpl->addOptimizationProfile(profile);

9451

}

9452

9461

int32 t getNbOptimizationProfiles() const noexcept

9462

{

9463

return mImpl->getNbOptimizationProfiles();

9464

}

9465

9473

void setProfilingVerbosity(ProfilingVerbosity verbosity) noexcept

9474

{

9475

mImpl->setProfilingVerbosity(verbosity);

9476

}

9477

9486

ProfilingVerbosity getProfilingVerbosity() const noexcept

9487

{

9488

return mImpl->getProfilingVerbosity();

9489

}

9490

9495

void setAlgorithmSelector(IAlgorithmSelector* selector) noexcept

9496

{

9497

mImpl->setAlgorithmSelector(selector);

9498

}

9499

9503

IAlgorithmSelector_* getAlgorithmSelector() const noexcept

9504

{

9505

return mImpl->getAlgorithmSelector();

9506

}

9507

9518

bool setCalibrationProfile(IOptimizationProfile const* profile) noexcept

9519

{

9520

return mImpl->setCalibrationProfile(profile);

9521

}

9522

9528

IOptimizationProfile const_* getCalibrationProfile() noexcept

9529

{

9530

return mImpl->getCalibrationProfile();

9531

}

9532

9545

void setQuantizationFlags(QuantizationFlags flags) noexcept

9546

{

9547

mImpl->setQuantizationFlags(flags);

9548

}

9549

9557

QuantizationFlags getQuantizationFlags() const noexcept

9558

{

9559

return mImpl->getQuantizationFlags();

9560

}

9561

9569

void clearQuantizationFlag(QuantizationFlag flag) noexcept

9570

{

9571

mImpl->clearQuantizationFlag(flag);

9572

}

9573

9581

void setQuantizationFlag(QuantizationFlag flag) noexcept

9582

{

9583

mImpl->setQuantizationFlag(flag);

9584

}

9585

9593

bool getQuantizationFlag(QuantizationFlag flag) const noexcept

9594

{

9595

return mImpl->getQuantizationFlag(flag);

9596

}

9597

9615

bool setTacticSources(TacticSources tacticSources) noexcept

9616

{

9617

return mImpl->setTacticSources(tacticSources);

9618

}

9619

9630

TacticSources getTacticSources() const noexcept

9631

{

9632

return mImpl->getTacticSources();

10.4 NvInfer.h

739

9633

}

9634

9649

nvinfer1::ITimingCache_* createTimingCache(void const* blob, std::size t size) const noexcept

9650

{

9651

return mImpl->createTimingCache(blob, size);

9652

}

9653

9672

bool setTimingCache(ITimingCache const& cache, bool ignoreMismatch) noexcept

9673

{

9674

return mImpl->setTimingCache(cache, ignoreMismatch);

9675

}

9676

9682

nvinfer1::ITimingCache const_* getTimingCache() const noexcept

9683

{

9684

return mImpl->getTimingCache();

9685

}

9686

9714

void setMemoryPoolLimit(MemoryPoolType pool, std::size t poolSize) noexcept

9715

{

9716

mImpl->setMemoryPoolLimit(pool, poolSize);

9717

}

9718

9733

std::size t getMemoryPoolLimit(MemoryPoolType pool) const noexcept

9734

{

9735

return mImpl->getMemoryPoolLimit(pool);

9736

}

9737

9751

void setPreviewFeature(PreviewFeature feature, bool enable) noexcept

9752

{

9753

mImpl->setPreviewFeature(feature, enable);

9754

}

9755

9765

bool getPreviewFeature(PreviewFeature feature) const noexcept

9766

{

9767

return mImpl->getPreviewFeature(feature);

9768

}

9769

9785

void setBuilderOptimizationLevel(int32 t level) noexcept

9786

{

9787

mImpl->setBuilderOptimizationLevel(level);

9788

}

9789

9797

int32 t getBuilderOptimizationLevel() noexcept

9798

{

9799

return mImpl->getBuilderOptimizationLevel();

9800

}

9801

9813

void setHardwareCompatibilityLevel(HardwareCompatibilityLevel hardwareCompatibilityLevel) noexcept

9814

{

9815

mImpl->setHardwareCompatibilityLevel(hardwareCompatibilityLevel);

9816

}

9817

9826

HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept

9827

{

9828

return mImpl->getHardwareCompatibilityLevel();

9829

}

9830

9839

void setPluginsToSerialize(char const_* const* paths, int32 t nbPaths) noexcept

9840

{

9841

mImpl->setPluginsToSerialize(paths, nbPaths);

9842

}

9843

9852

char const_* getPluginToSerialize(int32 t index) const noexcept

9853

{

9854

return mImpl->getPluginToSerialize(index);

9855

}

9856

9862

int32 t getNbPluginsToSerialize() const noexcept

9863

{

9864

return mImpl->getNbPluginsToSerialize();

9865

}

9866

9891

void setMaxAuxStreams(int32 t nbStreams) noexcept

9892

{

9893

mImpl->setMaxAuxStreams(nbStreams);

9894

}

9895

9901

int32 t getMaxAuxStreams() const noexcept

9902

{

9903

return mImpl->getMaxAuxStreams();

9904

}

740

File Documentation

9905

9906 protected:

9907

apiv::VBuilderConfig* mImpl;

9908 };

9909

9916 using NetworkDefinitionCreationFlags = uint32 t;

9917

9926 enum class NetworkDefinitionCreationFlag : int32 t

9927 {

9933

kEXPLICIT BATCH = 0,

9934

9937

kEXPLICIT PRECISION TRT DEPRECATED ENUM = 1,

9938 };

9939

9945 template <>

9946 constexpr inline int32 t EnumMax<NetworkDefinitionCreationFlag>() noexcept

9947 {

9948

return 2;

9949 }

9950

9958 class IBuilder : public INoCopy

9959 {

9960 public:

9961

virtual ˜IBuilder() noexcept = default;

9962

9973

TRT DEPRECATED void setMaxBatchSize(int32 t batchSize) noexcept

9974

{

9975

mImpl->setMaxBatchSize(batchSize);

9976

}

9977

9988

TRT DEPRECATED int32 t getMaxBatchSize() const noexcept

9989

{

9990

return mImpl->getMaxBatchSize();

9991

}

9992

9996

bool platformHasFastFp16() const noexcept

9997

{

9998

return mImpl->platformHasFastFp16();

9999

}

10000

10004

bool platformHasFastInt8() const noexcept

10005

{

10006

return mImpl->platformHasFastInt8();

10007

}

10008

10016

TRT DEPRECATED void destroy() noexcept

10017

{

10018

delete this;

10019

}

10020

10028

int32 t getMaxDLABatchSize() const noexcept

10029

{

10030

return mImpl->getMaxDLABatchSize();

10031

}

10032

10036

int32 t getNbDLACores() const noexcept

10037

{

10038

return mImpl->getNbDLACores();

10039

}

10040

10052

void setGpuAllocator(IGpuAllocator* allocator) noexcept

10053

{

10054

mImpl->setGpuAllocator(allocator);

10055

}

10056

10062

nvinfer1::IBuilderConfig_* createBuilderConfig() noexcept

10063

{

10064

return mImpl->createBuilderConfig();

10065

}

10066

10077

TRT DEPRECATED nvinfer1::ICudaEngine_* buildEngineWithConfig(

10078

INetworkDefinition& network, IBuilderConfig& config) noexcept

10079

{

10080

return mImpl->buildEngineWithConfig(network, config);

10081

}

10082

10095

nvinfer1::INetworkDefinition_* createNetworkV2(NetworkDefinitionCreationFlags flags) noexcept

10096

{

10097

return mImpl->createNetworkV2(flags);

10098

}

10099

10.4 NvInfer.h

741

10109

nvinfer1::IOptimizationProfile_* createOptimizationProfile() noexcept

10110

{

10111

return mImpl->createOptimizationProfile();

10112

}

10113

10125

10128

void setErrorRecorder(IErrorRecorder* recorder) noexcept

10129

{

10130

mImpl->setErrorRecorder(recorder);

10131

}

10132

10143

IErrorRecorder_* getErrorRecorder() const noexcept

10144

{

10145

return mImpl->getErrorRecorder();

10146

}

10147

10151

void reset() noexcept

10152

{

10153

mImpl->reset();

10154

}

10155

10159

bool platformHasTf32() const noexcept

10160

{

10161

return mImpl->platformHasTf32();

10162

}

10163

10178

nvinfer1::IHostMemory_* buildSerializedNetwork(INetworkDefinition& network, IBuilderConfig& config)

noexcept

10179

{

10180

return mImpl->buildSerializedNetwork(network, config);

10181

}

10182

10202

bool isNetworkSupported(INetworkDefinition const& network, IBuilderConfig const& config) const

noexcept

10203

{

10204

return mImpl->isNetworkSupported(network, config);

10205

}

10206

10212

ILogger_* getLogger() const noexcept

10213

{

10214

return mImpl->getLogger();

10215

}

10216

10226

bool setMaxThreads(int32 t maxThreads) noexcept

10227

{

10228

return mImpl->setMaxThreads(maxThreads);

10229

}

10230

10240

int32 t getMaxThreads() const noexcept

10241

{

10242

return mImpl->getMaxThreads();

10243

}

10244

10250

IPluginRegistry& getPluginRegistry() noexcept

10251

{

10252

return mImpl->getPluginRegistry();

10253

}

10254

10255

protected:

10256

apiv::VBuilder* mImpl;

10257

};

10258

10259

} // namespace nvinfer1

10260

10265

extern "C" TENSORRTAPI void* createInferBuilder INTERNAL(void* logger, int32 t version) noexcept;

10266

10267

namespace nvinfer1

10268

{

10269

namespace

10270

{

10271

10279

inline IBuilder* createInferBuilder(ILogger& logger) noexcept

10280

{

10281

return static cast<IBuilder*>(createInferBuilder INTERNAL(&logger, NV TENSORRT VERSION));

10282

}

10283

10284

} // namespace

10285

10298

extern "C" TENSORRTAPI nvinfer1::IPluginRegistry_* getBuilderPluginRegistry(

10299

nvinfer1::EngineCapability capability) noexcept;

10300

742

File Documentation

10301 namespace safe

10302 {

10304 class IPluginRegistry;

10305 } // namespace safe

10306

10314 extern "C" TENSORRTAPI nvinfer1::safe::IPluginRegistry_* getBuilderSafePluginRegistry(

10315

nvinfer1::EngineCapability capability) noexcept;

10316

10317 } // namespace nvinfer1

10318

10319 #endif // NV INFER H

10.5

NvInferConsistency.h File Reference

#include "NvInferConsistencyImpl.h"

#include "NvInferRuntimeBase.h"

#include "NvInferRuntimePlugin.h"

Classes

• class nvinfer1::consistency::IConsistencyChecker

Validates a serialized engine blob.

• class nvinfer1::consistency::IPluginChecker

Consistency Checker plugin class for user implemented Plugins.

Namespaces

• namespace nvinfer1

The TensorRT API version 1 namespace.

• namespace nvinfer1::consistency

Functions

• void ∗ createConsistencyChecker INTERNAL (void ∗logger, void const ∗blob, size t size, int32 t version)

Internal C entry point for creating IConsistencyChecker.

10.5.1

Function Documentation

10.5.1.1

createConsistencyChecker INTERNAL()

void ∗ createConsistencyChecker INTERNAL (

void ∗ logger,

void const ∗ blob,

size t size,

int32 t version )

Internal C entry point for creating IConsistencyChecker.

10.6 NvInferConsistency.h

743

10.6

NvInferConsistency.h

Go to the documentation of this file.

1 /*

* SPDX-License-Identifier: LicenseRef-NvidiaProprietary

* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual

* property and proprietary rights in and to this material, related

* documentation and any modifications thereto. Any use, reproduction,

* disclosure or distribution of this material and related documentation

* without an express license agreement from NVIDIA CORPORATION or

* its affiliates is strictly prohibited.

#ifndef NV INFER CONSISTENCY H

#define NV INFER CONSISTENCY H

#include "NvInferConsistencyImpl.h"

#include "NvInferRuntimeBase.h"

#include "NvInferRuntimePlugin.h"

namespace nvinfer1

{

namespace consistency

{

class IConsistencyChecker

{

public:

bool validate() const noexcept

{

return mImpl->validate();

}

virtual ˜IConsistencyChecker() = default;

protected:

apiv::VConsistencyChecker* mImpl;

IConsistencyChecker() = default;

IConsistencyChecker(IConsistencyChecker const& other) = delete;

IConsistencyChecker& operator=(IConsistencyChecker const& other) = delete;

IConsistencyChecker(IConsistencyChecker&& other) = delete;

IConsistencyChecker& operator=(IConsistencyChecker&& other) = delete;

};

class IPluginChecker : public IPluginCreator

77 {

public:

virtual bool validate(char const* name, void const* serialData, size t serialLength, PluginTensorDesc

const* in,

size t nbInputs, PluginTensorDesc const* out, size t nbOutputs, int64 t workspaceSize) const noexcept

= 0;

IPluginChecker() = default;

virtual ˜IPluginChecker() override = default;

protected:

100

IPluginChecker(IPluginChecker const&) = default;

101

IPluginChecker(IPluginChecker&&) = default;

102

IPluginChecker& operator=(IPluginChecker const&) & = default;

103

IPluginChecker& operator=(IPluginChecker&&) & = default;

104 };

105

106 } // namespace consistency

107

108 } // namespace nvinfer1

109

110 extern "C" TENSORRTAPI void_* createConsistencyChecker INTERNAL(void* logger, void const* blob, size t size,

111

int32 t version);

112

113 namespace nvinfer1

114 {

115

116 namespace consistency

117 {

744

File Documentation

125 namespace // anonymous

126 {

127

128 inline IConsistencyChecker* createConsistencyChecker(ILogger& logger, void const* blob, size t size)

129 {

130

return static cast<IConsistencyChecker*>(

131

createConsistencyChecker INTERNAL(&logger, blob, size, NV TENSORRT VERSION));

132 }

133

134 } // namespace

135

136 } // namespace consistency

137

138 } // namespace nvinfer1

139

140 #endif // NV INFER CONSISTENCY H

10.7

NvInferLegacyDims.h File Reference

#include "NvInferRuntimeCommon.h"

Classes

• class nvinfer1::Dims2

Descriptor for two-dimensional data.

• class nvinfer1::DimsHW

Descriptor for two-dimensional spatial data.

• class nvinfer1::Dims3

Descriptor for three-dimensional data.

• class nvinfer1::Dims4

Descriptor for four-dimensional data.

Namespaces

• namespace nvinfer1

The TensorRT API version 1 namespace.

10.7.1

Detailed Description

This file contains declarations of legacy dimensions types which use channel semantics in their names, and declarations

on which those types rely.

10.8 NvInferLegacyDims.h

745

10.8

NvInferLegacyDims.h

Go to the documentation of this file.

1 /*

* SPDX-License-Identifier: LicenseRef-NvidiaProprietary

* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual

* property and proprietary rights in and to this material, related

* documentation and any modifications thereto. Any use, reproduction,

* disclosure or distribution of this material and related documentation

* without an express license agreement from NVIDIA CORPORATION or

* its affiliates is strictly prohibited.

#ifndef NV INFER LEGACY DIMS H

#define NV INFER LEGACY DIMS H

#include "NvInferRuntimeCommon.h"

namespace nvinfer1

{

class Dims2 : public Dims

{

public:

Dims2()

: Dims2(0, 0)

{

}

Dims2(int32 t d0, int32 t d1)

{

nbDims = 2;

d[0] = d0;

d[1] = d1;

for (int32 t i{nbDims}; i < Dims::MAX DIMS; ++i)

{

d[i] = 0;

}

};

class DimsHW : public Dims2

{

public:

DimsHW()

: Dims2()

{

}

DimsHW(int32 t height, int32 t width)

: Dims2(height, width)

{

}

int32 t& h()

{

return d[0];

}

100

106

int32 t h() const

107

{

108

return d[0];

109

}

110

116

int32 t& w()

117

{

118

return d[1];

119

}

120

126

int32 t w() const

127

{

128

return d[1];

129

}

130 };

131

136 class Dims3 : public Dims2

137 {

746

File Documentation

138 public:

142

Dims3()

143

: Dims3(0, 0, 0)

144

{

145

}

146

154

Dims3(int32 t d0, int32 t d1, int32 t d2)

155

: Dims2(d0, d1)

156

{

157

nbDims = 3;

158

d[2] = d2;

159

}

160 };

161

166 class Dims4 : public Dims3

167 {

168 public:

172

Dims4()

173

: Dims4(0, 0, 0, 0)

174

{

175

}

176

185

Dims4(int32 t d0, int32 t d1, int32 t d2, int32 t d3)

186

: Dims3(d0, d1, d2)

187

{

188

nbDims = 4;

189

d[3] = d3;

190

}

191 };

192

193 } // namespace nvinfer1

194

195 #endif // NV INFER LEGCY DIMS H

10.9

NvInferPlugin.h File Reference

#include "NvInfer.h"

#include "NvInferPluginUtils.h"

Functions

• TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createRPNROIPlugin (int32 t featureStride, int32 t preNms←↩

Top, int32 t nmsMaxOut, float iouThreshold, float minBoxSize, float spatialScale, nvinfer1::DimsHW pooling,

nvinfer1::Weights anchorRatios, nvinfer1::Weights anchorScales)

Create a plugin layer that fuses the RPN and ROI pooling using user-defined parameters. Registered plugin type ”←↩

RPROI TRT”. Registered plugin version ”1”.

• TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createNormalizePlugin (nvinfer1::Weights const ∗scales, bool

acrossSpatial, bool channelShared, float eps)

The Normalize plugin layer normalizes the input to have L2 norm of 1 with scale learnable. Registered plugin type

”Normalize TRT”. Registered plugin version ”1”.

• TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createPriorBoxPlugin (nvinfer1::plugin::PriorBoxParameters

param)

The PriorBox plugin layer generates the prior boxes of designated sizes and aspect ratios across all dimensions (H

x W). PriorBoxParameters defines a set of parameters for creating the PriorBox plugin layer. Registered plugin type

”PriorBox TRT”. Registered plugin version ”1”.

• TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createAnchorGeneratorPlugin (nvinfer1::plugin::GridAnchorParameters

∗param, int32 t numLayers)

The Grid Anchor Generator plugin layer generates the prior boxes of designated sizes and aspect ratios across all

dimensions (H x W) for all feature maps. GridAnchorParameters defines a set of parameters for creating the Grid←↩

AnchorGenerator plugin layer. Registered plugin type ”GridAnchor TRT”. Registered plugin version ”1”.

10.9 NvInferPlugin.h File Reference

747

•

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createNMSPlugin (nvinfer1::plugin::DetectionOutputParameters

param)

The DetectionOutput plugin layer generates the detection output based on location and confidence predictions by doing

non maximum suppression. DetectionOutputParameters defines a set of parameters for creating the DetectionOutput

plugin layer. Registered plugin type ”NMS TRT”. Registered plugin version ”1”.

•

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createReorgPlugin (int32 t stride)

The Reorg plugin reshapes input of shape CxHxW into a (C∗stride∗stride)x(H/stride)x(W/stride) shape, used in YOLOv2.

It does that by taking 1 x stride x stride slices from tensor and flattening them into (stride x stride) x 1 x 1 shape. Registered

plugin type ”Reorg TRT”. Registered plugin version ”1”.

•

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createRegionPlugin (nvinfer1::plugin::RegionParameters

params)

The Region plugin layer performs region proposal calculation: generate 5 bounding boxes per cell (for yolo9000, gener-

ate 3 bounding boxes per cell). For each box, calculating its probablities of objects detections from 80 pre-defined clas-

sifications (yolo9000 has 9416 pre-defined classifications, and these 9416 items are organized as work-tree structure).

RegionParameters defines a set of parameters for creating the Region plugin layer. Registered plugin type ”Region TRT”.

Registered plugin version ”1”.

•

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createBatchedNMSPlugin (nvinfer1::plugin::NMSParameters

param)

The BatchedNMS Plugin performs non max suppression on the input boxes, per batch, across all classes. It greedily

selects a subset of bounding boxes in descending order of score. Prunes away boxes that have a high intersection-over-

union (IOU) overlap with previously selected boxes. Bounding boxes are supplied as [y1, x1, y2, x2], where (y1, x1) and

(y2, x2) are the coordinates of any diagonal pair of box corners and the coordinates can be provided as normalized (i.e.,

lying in the interval [0, 1]) or absolute. The plugin expects two inputs. Input0 is expected to be 4-D float boxes tensor of

shape [batch size, num boxes, q, 4], where q can be either 1 (if shareLocation is true) or num classes. Input1 is expected

to be a 3-D float scores tensor of shape [batch size, num boxes, num classes] representing a single score corresponding

to each box. The plugin returns four outputs. num detections : A [batch size] int32 tensor indicating the number of valid

detections per batch item. Can be less than keepTopK. Only the top num detections[i] entries in nmsed boxes[i], nmsed←↩

scores[i] and nmsed classes[i] are valid. nmsed boxes : A [batch size, max detections, 4] float32 tensor containing the

co-ordinates of non-max suppressed boxes. nmsed scores : A [batch size, max detections] float32 tensor containing the

scores for the boxes. nmsed classes : A [batch size, max detections] float32 tensor containing the classes for the boxes.

•

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createSplitPlugin (int32 t axis, int32 t ∗output lengths, int32←↩

t noutput)

The Split Plugin performs a split operation on the input tensor. It splits the input tensor into several output tensors, each

of a length corresponding to output lengths. The split occurs along the axis specified by axis.

•

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createInstanceNormalizationPlugin (float epsilon, nvinfer1::Weights

scale weights, nvinfer1::Weights bias weights)

The Instance Normalization Plugin computes the instance normalization of an input tensor. The instance normalization

is calculated as found in the paper https://arxiv.org/abs/1607.08022. The calculation is y = scale ∗ (x -

mean) / sqrt(variance + epsilon) + bias where mean and variance are computed per instance per channel.

•

bool initLibNvInferPlugins (void ∗logger, char const ∗libNamespace)

Initialize and register all the existing TensorRT plugins to the Plugin Registry with an optional namespace. The plugin

library author should ensure that this function name is unique to the library. This function should be called once before

accessing the Plugin Registry.

10.9.1

Detailed Description

This is the API for the Nvidia provided TensorRT plugins.

10.9.2

Function Documentation

748

File Documentation

10.9.2.1

createAnchorGeneratorPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createAnchorGeneratorPlugin (

nvinfer1::plugin::GridAnchorParameters ∗ param,

int32 t numLayers )

The Grid Anchor Generator plugin layer generates the prior boxes of designated sizes and aspect ratios across all

dimensions (H x W) for all feature maps. GridAnchorParameters defines a set of parameters for creating the Grid←↩

AnchorGenerator plugin layer. Registered plugin type ”GridAnchor TRT”. Registered plugin version ”1”.

Deprecated Deprecated in TensorRT 8.5. Use GridAnchorPluginCreator::createPlugin() to create an instance of ”←↩

GridAnchor TRT” version 1 plugin.

10.9.2.2

createBatchedNMSPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createBatchedNMSPlugin (

nvinfer1::plugin::NMSParameters param )

The BatchedNMS Plugin performs non max suppression on the input boxes, per batch, across all classes. It greedily

selects a subset of bounding boxes in descending order of score. Prunes away boxes that have a high intersection-over-

union (IOU) overlap with previously selected boxes. Bounding boxes are supplied as [y1, x1, y2, x2], where (y1, x1)

and (y2, x2) are the coordinates of any diagonal pair of box corners and the coordinates can be provided as normalized

(i.e., lying in the interval [0, 1]) or absolute. The plugin expects two inputs. Input0 is expected to be 4-D float boxes

tensor of shape [batch size, num boxes, q, 4], where q can be either 1 (if shareLocation is true) or num classes. Input1

is expected to be a 3-D float scores tensor of shape [batch size, num boxes, num classes] representing a single score

corresponding to each box. The plugin returns four outputs. num detections : A [batch size] int32 tensor indicating

the number of valid detections per batch item. Can be less than keepTopK. Only the top num detections[i] entries

in nmsed boxes[i], nmsed scores[i] and nmsed classes[i] are valid. nmsed boxes : A [batch size, max detections,

4] float32 tensor containing the co-ordinates of non-max suppressed boxes. nmsed scores : A [batch size, max ←↩

detections] float32 tensor containing the scores for the boxes. nmsed classes : A [batch size, max detections] float32

tensor containing the classes for the boxes.

Registered plugin type ”BatchedNMS TRT”. Registered plugin version ”1”.

The batched NMS plugin can require a lot of workspace due to intermediate buffer usage. To get the estimated

workspace size for the plugin for a batch size, use the API plugin->getWorkspaceSize(batchSize).

Deprecated Deprecated in TensorRT 8.5. Use BatchedNMSPluginCreator::createPlugin() to create an instance of

”BatchedNMS TRT” version 1 plugin.

10.9.2.3

createInstanceNormalizationPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createInstanceNormalizationPlugin (

float epsilon,

nvinfer1::Weights scale weights,

nvinfer1::Weights bias weights )

The Instance Normalization Plugin computes the instance normalization of an input tensor. The instance normalization

is calculated as found in the paper https://arxiv.org/abs/1607.08022. The calculation is y = scale ∗ (x

- mean) / sqrt(variance + epsilon) + bias where mean and variance are computed per instance per channel.

10.9 NvInferPlugin.h File Reference

749

Parameters

epsilon

The epsilon value to use to avoid division by zero.

scale weights

The input 1-dimensional scale weights of size C to scale.

bias weights

The input 1-dimensional bias weights of size C to offset.

Deprecated Deprecated in TensorRT 8.5. Use InstanceNormalizationPluginCreator::createPlugin() to create an in-

stance of ”InstanceNormalization TRT” version 1 plugin.

10.9.2.4

createNMSPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createNMSPlugin (

nvinfer1::plugin::DetectionOutputParameters param )

The DetectionOutput plugin layer generates the detection output based on location and confidence predictions by doing

non maximum suppression. DetectionOutputParameters defines a set of parameters for creating the DetectionOutput

plugin layer. Registered plugin type ”NMS TRT”. Registered plugin version ”1”.

Deprecated Deprecated in TensorRT 8.5. Use NMSPluginCreator::createPlugin() to create an instance of ”NMS ←↩

TRT” version 1 plugin.

10.9.2.5

createNormalizePlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createNormalizePlugin (

nvinfer1::Weights const ∗ scales,

bool acrossSpatial,

bool channelShared,

float eps )

The Normalize plugin layer normalizes the input to have L2 norm of 1 with scale learnable. Registered plugin type

”Normalize TRT”. Registered plugin version ”1”.

Parameters

scales

Scale weights that are applied to the output tensor.

acrossSpatial

Whether to compute the norm over adjacent channels (acrossSpatial is true) or nearby spatial

locations (within channel in which case acrossSpatial is false).

channelShared

Whether the scale weight(s) is shared across channels.

eps

Epsilon for not dividing by zero.

750

File Documentation

Deprecated Deprecated in TensorRT 8.5. Use NormalizePluginCreator::createPlugin() to create an instance of ”←↩

Normalize TRT” version 1 plugin.

10.9.2.6

createPriorBoxPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createPriorBoxPlugin (

nvinfer1::plugin::PriorBoxParameters param )

The PriorBox plugin layer generates the prior boxes of designated sizes and aspect ratios across all dimensions (H

x W). PriorBoxParameters defines a set of parameters for creating the PriorBox plugin layer. Registered plugin type

”PriorBox TRT”. Registered plugin version ”1”.

Deprecated Deprecated in TensorRT 8.5. Use PriorBoxPluginCreator::createPlugin() to create an instance of ”Prior←↩

Box TRT” version 1 plugin.

10.9.2.7

createRegionPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createRegionPlugin (

nvinfer1::plugin::RegionParameters params )

The Region plugin layer performs region proposal calculation: generate 5 bounding boxes per cell (for yolo9000,

generate 3 bounding boxes per cell). For each box, calculating its probablities of objects detections from 80 pre-

defined classifications (yolo9000 has 9416 pre-defined classifications, and these 9416 items are organized as work-tree

structure). RegionParameters defines a set of parameters for creating the Region plugin layer. Registered plugin type

”Region TRT”. Registered plugin version ”1”.

Deprecated Deprecated in TensorRT 8.5. Use RegionPluginCreator::createPlugin() to create an instance of ”Region←↩

TRT” version 1 plugin.

10.9.2.8

createReorgPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createReorgPlugin (

int32 t stride )

The Reorg plugin reshapes input of shape CxHxW into a (C∗stride∗stride)x(H/stride)x(W/stride) shape, used in

YOLOv2. It does that by taking 1 x stride x stride slices from tensor and flattening them into (stride x stride) x 1

x 1 shape. Registered plugin type ”Reorg TRT”. Registered plugin version ”1”.

10.9 NvInferPlugin.h File Reference

751

Parameters

stride

Strides in H and W, it should divide both H and W. Also stride ∗ stride should be less than or equal to C.

Deprecated Deprecated in TensorRT 8.5. Use ReorgPluginCreator::createPlugin() to create an instance of ”Reorg ←↩

TRT” version 1 plugin.

10.9.2.9

createRPNROIPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createRPNROIPlugin (

int32 t featureStride,

int32 t preNmsTop,

int32 t nmsMaxOut,

float iouThreshold,

float minBoxSize,

float spatialScale,

nvinfer1::DimsHW pooling,

nvinfer1::Weights anchorRatios,

nvinfer1::Weights anchorScales )

Create a plugin layer that fuses the RPN and ROI pooling using user-defined parameters. Registered plugin type

”RPROI TRT”. Registered plugin version ”1”.

Parameters

featureStride

Feature stride.

preNmsTop

Number of proposals to keep before applying NMS.

nmsMaxOut

Number of remaining proposals after applying NMS.

iouThreshold

IoU threshold.

minBoxSize

Minimum allowed bounding box size before scaling.

spatialScale

Spatial scale between the input image and the last feature map.

pooling

Spatial dimensions of pooled ROIs.

anchorRatios

Aspect ratios for generating anchor windows.

anchorScales

Scales for generating anchor windows.

Returns

Returns a FasterRCNN fused RPN+ROI pooling plugin. Returns nullptr on invalid inputs.

Deprecated Deprecated in TensorRT 8.5. Use RPROIPluginCreator::createPlugin() to create an instance of ”RPROI←↩

TRT” version 1 plugin.

752

File Documentation

10.9.2.10 createSplitPlugin()

TRT DEPRECATED API nvinfer1::IPluginV2 ∗ createSplitPlugin (

int32 t axis,

int32 t ∗ output lengths,

int32 t noutput )

The Split Plugin performs a split operation on the input tensor. It splits the input tensor into several output tensors,

each of a length corresponding to output lengths. The split occurs along the axis specified by axis.

Parameters

axis

The axis to split on.

output lengths

The lengths of the output tensors.

noutput

The number of output tensors.

Deprecated Deprecated in TensorRT 8.5 along with the ”Split” plugin. Use INetworkDefinition::addSlice() to add

slice layer(s) as necessary to accomplish the required effect.

10.9.2.11 initLibNvInferPlugins()

bool initLibNvInferPlugins (

void ∗ logger,

char const ∗ libNamespace )

Initialize and register all the existing TensorRT plugins to the Plugin Registry with an optional namespace. The plugin

library author should ensure that this function name is unique to the library. This function should be called once before

accessing the Plugin Registry.

Parameters

logger

Logger object to print plugin registration information

libNamespace

Namespace used to register all the plugins in this library

10.10

NvInferPlugin.h

Go to the documentation of this file.

1 /*

* SPDX-License-Identifier: LicenseRef-NvidiaProprietary

* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual

* property and proprietary rights in and to this material, related

* documentation and any modifications thereto. Any use, reproduction,

10.11 NvInferPluginUtils.h File Reference

753

* disclosure or distribution of this material and related documentation

* without an express license agreement from NVIDIA CORPORATION or

* its affiliates is strictly prohibited.

13 #ifndef NV INFER PLUGIN H

14 #define NV INFER PLUGIN H

16 #include "NvInfer.h"

17 #include "NvInferPluginUtils.h"

24 extern "C"

25 {

TRT DEPRECATED API nvinfer1::IPluginV2_* createRPNROIPlugin(int32 t featureStride, int32 t preNmsTop,

int32 t nmsMaxOut,

float iouThreshold, float minBoxSize, float spatialScale, nvinfer1::DimsHW pooling,

nvinfer1::Weights anchorRatios, nvinfer1::Weights anchorScales);

TRT DEPRECATED API nvinfer1::IPluginV2_* createNormalizePlugin(

nvinfer1::Weights const* scales, bool acrossSpatial, bool channelShared, float eps);

TRT DEPRECATED API nvinfer1::IPluginV2_* createPriorBoxPlugin(nvinfer1::plugin::PriorBoxParameters param);

TRT DEPRECATED API nvinfer1::IPluginV2_* createAnchorGeneratorPlugin(

nvinfer1::plugin::GridAnchorParameters* param, int32 t numLayers);

TRT DEPRECATED API nvinfer1::IPluginV2_* createNMSPlugin(nvinfer1::plugin::DetectionOutputParameters

param);

105

TRT DEPRECATED API nvinfer1::IPluginV2_* createReorgPlugin(int32 t stride);

106

117

TRT DEPRECATED API nvinfer1::IPluginV2_* createRegionPlugin(nvinfer1::plugin::RegionParameters params);

118

151

TRT DEPRECATED API nvinfer1::IPluginV2_* createBatchedNMSPlugin(nvinfer1::plugin::NMSParameters param);

152

164

TRT DEPRECATED API nvinfer1::IPluginV2_* createSplitPlugin(int32 t axis, int32 t* output lengths, int32 t

noutput);

165

178

TRT DEPRECATED API nvinfer1::IPluginV2_* createInstanceNormalizationPlugin(

179

float epsilon, nvinfer1::Weights scale weights, nvinfer1::Weights bias weights);

180

188

TENSORRTAPI bool initLibNvInferPlugins(void* logger, char const* libNamespace);

189

190 } // extern "C"

191

192 #endif // NV INFER PLUGIN H

10.11

NvInferPluginUtils.h File Reference

#include "NvInferRuntimeCommon.h"

Classes

• struct nvinfer1::plugin::Quadruple

The Permute plugin layer permutes the input tensor by changing the memory order of the data. Quadruple defines a

structure that contains an array of 4 integers. They can represent the permute orders or the strides in each dimension.

• struct nvinfer1::plugin::PriorBoxParameters

The PriorBox plugin layer generates the prior boxes of designated sizes and aspect ratios across all dimensions (H x W).

PriorBoxParameters defines a set of parameters for creating the PriorBox plugin layer. It contains:

• struct nvinfer1::plugin::RPROIParams

RPROIParams is used to create the RPROIPlugin instance. It contains:

• struct nvinfer1::plugin::GridAnchorParameters

754

File Documentation

The Anchor Generator plugin layer generates the prior boxes of designated sizes and aspect ratios across all dimen-

sions (H x W). GridAnchorParameters defines a set of parameters for creating the plugin layer for all feature maps. It

contains:

• struct nvinfer1::plugin::DetectionOutputParameters

The DetectionOutput plugin layer generates the detection output based on location and confidence predictions by doing

non maximum suppression. This plugin first decodes the bounding boxes based on the anchors generated. It then

performs non max suppression on the decoded bounding boxes. DetectionOutputParameters defines a set of parameters

for creating the DetectionOutput plugin layer. It contains:

• struct nvinfer1::plugin::softmaxTree

When performing yolo9000, softmaxTree is helping to do softmax on confidence scores, for element to get the precise

classification through word-tree structured classification definition.

• struct nvinfer1::plugin::RegionParameters

The Region plugin layer performs region proposal calculation: generate 5 bounding boxes per cell (for yolo9000, gen-

erate 3 bounding boxes per cell). For each box, calculating its probablities of objects detections from 80 pre-defined

classifications (yolo9000 has 9418 pre-defined classifications, and these 9418 items are organized as work-tree struc-

ture). RegionParameters defines a set of parameters for creating the Region plugin layer.

• struct nvinfer1::plugin::NMSParameters

The NMSParameters are used by the BatchedNMSPlugin for performing the non max suppression operation over boxes

for object detection networks.

Namespaces

• namespace nvinfer1

The TensorRT API version 1 namespace.

• namespace nvinfer1::plugin

Enumerations

• enum class nvinfer1::plugin::CodeTypeSSD : int32 t { nvinfer1::plugin::CORNER = 0 , nvinfer1::plugin::CENTER SIZE

= 1 , nvinfer1::plugin::CORNER SIZE = 2 , nvinfer1::plugin::TF CENTER = 3 }

The type of encoding used for decoding the bounding boxes and loc data.

10.11.1

Detailed Description

This is the API for the Nvidia provided TensorRT plugin utilities. It lists all the parameters utilized by the TensorRT

plugins.

10.12

NvInferPluginUtils.h

Go to the documentation of this file.

1 /*

* SPDX-License-Identifier: LicenseRef-NvidiaProprietary

* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual

* property and proprietary rights in and to this material, related

* documentation and any modifications thereto. Any use, reproduction,

* disclosure or distribution of this material and related documentation

* without an express license agreement from NVIDIA CORPORATION or

10.12 NvInferPluginUtils.h

755

* its affiliates is strictly prohibited.

13 #ifndef NV INFER PLUGIN UTILS H

14 #define NV INFER PLUGIN UTILS H

16 #include "NvInferRuntimeCommon.h"

25 namespace nvinfer1

26 {

27 namespace plugin

28 {

35 typedef struct

36 {

int32 t data[4];

38 } Quadruple;

60 struct PriorBoxParameters

61 {

float *minSize, *maxSize, *aspectRatios;

int32 t numMinSize, numMaxSize, numAspectRatios;

bool flip;

bool clip;

float variance[4];

int32 t imgH, imgW;

float stepH, stepW;

float offset;

70 };

87 struct RPROIParams

88 {

int32 t poolingH;

int32 t poolingW;

int32 t featureStride;

int32 t preNmsTop;

int32 t nmsMaxOut;

int32 t anchorsRatioCount;

int32 t anchorsScaleCount;

float iouThreshold;

float minBoxSize;

float spatialScale;

99 };

100

101

114 struct GridAnchorParameters

115 {

116

float minSize, maxSize;

117

float_* aspectRatios;

118

int32 t numAspectRatios, H, W;

119

float variance[4];

120 };

121

126 enum class CodeTypeSSD : int32 t

127 {

128

CORNER = 0,

129

CENTER SIZE = 1,

130

CORNER SIZE = 2,

131

TF CENTER = 3

132 };

133

153 struct DetectionOutputParameters

154 {

155

bool shareLocation, varianceEncodedInTarget;

156

int32 t backgroundLabelId, numClasses, topK, keepTopK;

157

float confidenceThreshold, nmsThreshold;

158

CodeTypeSSD codeType;

159

int32 t inputOrder[3];

160

bool confSigmoid;

161

bool isNormalized;

162

bool isBatchAgnostic{true};

163 };

164

168 struct softmaxTree

169 {

170

int32 t* leaf;

171

int32 t n;

172

int32 t* parent;

173

int32 t* child;

174

int32 t* group;

756

File Documentation

175

char_** name;

176

177

int32 t groups;

178

int32 t* groupSize;

179

int32 t* groupOffset;

180 };

181

192 struct RegionParameters

193 {

194

int32 t num;

195

int32 t coords;

196

int32 t classes;

197

softmaxTree_* smTree;

198 };

199

216

217 struct NMSParameters

218 {

219

bool shareLocation;

220

int32 t backgroundLabelId, numClasses, topK, keepTopK;

221

float scoreThreshold, iouThreshold;

222

bool isNormalized;

223 };

224

225 } // namespace plugin

226 } // namespace nvinfer1

227

228 #endif // NV INFER PLUGIN UTILS H

10.13

NvInferRuntime.h File Reference

#include "NvInferImpl.h"

#include "NvInferRuntimeCommon.h"

Classes

• class nvinfer1::INoCopy

Forward declaration of IEngineInspector for use by other interfaces.

• struct nvinfer1::impl::EnumMaxImpl< EngineCapability >

Maximum number of elements in EngineCapability enum.

• class nvinfer1::Weights

An array of weights used as a layer parameter.

• class nvinfer1::IHostMemory

Class to handle library allocated memory that is accessible to the user.

• struct nvinfer1::impl::EnumMaxImpl< TensorLocation >

Maximum number of elements in TensorLocation enum.

• class nvinfer1::IDimensionExpr

• class nvinfer1::IExprBuilder

• class nvinfer1::DimsExprs

• class nvinfer1::DynamicPluginTensorDesc

• class nvinfer1::IPluginV2DynamicExt

• class nvinfer1::IProfiler

Application-implemented interface for profiling.

• class nvinfer1::IRuntime

Allows a serialized functionally unsafe engine to be deserialized.

• class nvinfer1::IRefitter

10.13 NvInferRuntime.h File Reference

757

Updates weights in an engine.

• class nvinfer1::IOptimizationProfile

Optimization profile for dynamic input dimensions and shape tensors.

• class nvinfer1::ICudaEngine

An engine for executing inference on a built network, with functionally unsafe features.

• class nvinfer1::IOutputAllocator

Callback from ExecutionContext::enqueueV3()

• class nvinfer1::IExecutionContext

Context for executing inference using an engine, with functionally unsafe features.

• class nvinfer1::IEngineInspector

An engine inspector which prints out the layer information of an engine or an execution context.

• class nvinfer1::PluginRegistrar< T >

Register the plugin creator to the registry The static registry object will be instantiated when the plugin library is loaded.

This static object will register all creators available in the library to the registry.

• class nvinfer1::ILoggerFinder

A virtual base class to find a logger. Allows a plugin to find an instance of a logger if it needs to emit a log message. A

pointer to an instance of this class is passed to a plugin shared library on initialization when that plugin is serialized as

part of a version-compatible plan. See the plugin chapter in the developer guide for details.

Namespaces

• namespace nvinfer1

The TensorRT API version 1 namespace.

• namespace nvinfer1::impl

Macros

• #define REGISTER TENSORRT PLUGIN(name)

static nvinfer1::PluginRegistrar<name> plugin←↩

Registrar##name {}

Typedefs

• using nvinfer1::TempfileControlFlags = uint32 t

Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operations.

• using nvinfer1::TacticSources = uint32 t

Represents a collection of one or more TacticSource values combine using bitwise-OR operations.

758

File Documentation

Enumerations

•

enum class nvinfer1::EngineCapability : int32 t {

nvinfer1::kSTANDARD

nvinfer1::kDEFAULT = kSTANDARD , nvinfer1::kSAFETY

nvinfer1::kSAFE GPU = kSAFETY ,

nvinfer1::kDLA STANDALONE = 2 , nvinfer1::kSAFE DLA = kDLA STANDALONE }

List of supported engine capability flows.

•

enum class nvinfer1::DimensionOperation : int32 t {

nvinfer1::kSUM = 0 , nvinfer1::kPROD = 1 , nvinfer1::kMAX = 2 , nvinfer1::kMIN = 3 ,

nvinfer1::kSUB = 4 , nvinfer1::kEQUAL = 5 , nvinfer1::kLESS = 6 , nvinfer1::kFLOOR DIV = 7 ,

nvinfer1::kCEIL DIV = 8 }

An operation on two IDimensionExpr, which represent integer expressions used in dimension computations.

•

enum class nvinfer1::TensorLocation : int32 t { nvinfer1::kDEVICE = 0 , nvinfer1::kHOST = 1 }

The location for tensor data storage, device or host.

•

enum class nvinfer1::WeightsRole : int32 t {

nvinfer1::kKERNEL = 0 , nvinfer1::kBIAS = 1 , nvinfer1::kSHIFT = 2 , nvinfer1::kSCALE = 3 ,

nvinfer1::kCONSTANT = 4 , nvinfer1::kANY = 5 }

How a layer uses particular Weights.

•

enum class nvinfer1::DeviceType : int32 t { nvinfer1::kGPU , nvinfer1::kDLA }

The device that this layer/network will execute on.

•

enum class nvinfer1::TempfileControlFlag :

int32 t { nvinfer1::kALLOW IN MEMORY FILES = 0 ,

nvinfer1::kALLOW TEMPORARY FILES = 1 }

Flags used to control TensorRT's behavior when creating executable temporary files.

•

enum class nvinfer1::OptProfileSelector : int32 t { nvinfer1::kMIN = 0 , nvinfer1::kOPT = 1 , nvinfer1::kMAX

=2}

When setting or querying optimization profile parameters (such as shape tensor inputs or dynamic dimensions), select

whether we are interested in the minimum, optimum, or maximum values for these parameters. The minimum and max-

imum specify the permitted range that is supported at runtime, while the optimum value is used for the kernel selection.

This should be the ”typical” value that is expected to occur at runtime.

•

enum class nvinfer1::TacticSource : int32 t {

nvinfer1::kCUBLAS = 0 , nvinfer1::kCUBLAS LT = 1 , nvinfer1::kCUDNN = 2 , nvinfer1::kEDGE MASK CONVOLUTIONS

=3,

nvinfer1::kJIT CONVOLUTIONS = 4 }

List of tactic sources for TensorRT.

•

enum class nvinfer1::ProfilingVerbosity : int32 t {

nvinfer1::kLAYER NAMES ONLY = 0 , nvinfer1::kNONE = 1 , nvinfer1::kDETAILED = 2 , nvinfer1::kDEFAULT

= kLAYER NAMES ONLY ,

nvinfer1::kVERBOSE = kDETAILED }

List of verbosity levels of layer information exposed in NVTX annotations and in IEngineInspector.

•

enum class nvinfer1::LayerInformationFormat : int32 t { nvinfer1::kONELINE = 0 , nvinfer1::kJSON = 1 }

The format in which the IEngineInspector prints the layer information.

Functions

• template<> constexpr int32 t nvinfer1::EnumMax< DimensionOperation > () noexcept

Maximum number of elements in DimensionOperation enum.

• template<> constexpr int32 t nvinfer1::EnumMax< WeightsRole > () noexcept

Maximum number of elements in WeightsRole enum.

• template<> constexpr int32 t nvinfer1::EnumMax< DeviceType > () noexcept

10.13 NvInferRuntime.h File Reference

759

Maximum number of elements in DeviceType enum.

• template<> constexpr int32 t nvinfer1::EnumMax< TempfileControlFlag > () noexcept

Maximum number of elements in TempfileControlFlag enum.

• template<> constexpr int32 t nvinfer1::EnumMax< OptProfileSelector > () noexcept

Number of different values of OptProfileSelector enum.

• template<> constexpr int32 t nvinfer1::EnumMax< TacticSource > () noexcept

Maximum number of tactic sources in TacticSource enum.

• template<> constexpr int32 t nvinfer1::EnumMax< ProfilingVerbosity > () noexcept

Maximum number of profile verbosity levels in ProfilingVerbosity enum.

• template<> constexpr int32 t nvinfer1::EnumMax< LayerInformationFormat > () noexcept

• nvinfer1::IPluginRegistry ∗ getPluginRegistry () noexcept

Return the plugin registry.

• nvinfer1::ILogger ∗ getLogger () noexcept

Return the logger object.

10.13.1

Detailed Description

This is the top-level API file for TensorRT extended runtime library.

10.13.2

Macro Definition Documentation

10.13.2.1 REGISTER TENSORRT PLUGIN

#define REGISTER TENSORRT PLUGIN(

name ) static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}

10.13.3

Function Documentation

10.13.3.1 getLogger()

nvinfer1::ILogger ∗ getLogger ( ) [noexcept]

Return the logger object.

Note

the global logger is used only by standalone functions which have no associated builder, runtime or refitter.

760

File Documentation

10.13.3.2 getPluginRegistry()

nvinfer1::IPluginRegistry ∗ getPluginRegistry ( ) [noexcept]

Return the plugin registry.

10.14

NvInferRuntime.h

Go to the documentation of this file.

1 /*

* SPDX-License-Identifier: LicenseRef-NvidiaProprietary

* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual

* property and proprietary rights in and to this material, related

* documentation and any modifications thereto. Any use, reproduction,

* disclosure or distribution of this material and related documentation

* without an express license agreement from NVIDIA CORPORATION or

* its affiliates is strictly prohibited.

#ifndef NV INFER RUNTIME H

#define NV INFER RUNTIME H

#include "NvInferImpl.h"

#include "NvInferRuntimeCommon.h"

namespace nvinfer1

{

class IExecutionContext;

class ICudaEngine;

class IPluginFactory;

class IEngineInspector;

class INoCopy

{

protected:

INoCopy() = default;

virtual ˜INoCopy() = default;

INoCopy(INoCopy const& other) = delete;

INoCopy& operator=(INoCopy const& other) = delete;

INoCopy(INoCopy&& other) = delete;

INoCopy& operator=(INoCopy&& other) = delete;

};

enum class EngineCapability : int32 t

{

kSTANDARD = 0,

kDEFAULT TRT DEPRECATED ENUM = kSTANDARD,

kSAFETY = 1,

kSAFE GPU TRT DEPRECATED ENUM = kSAFETY,

kDLA STANDALONE = 2,

kSAFE DLA TRT DEPRECATED ENUM = kDLA STANDALONE,

};

100

101 namespace impl

102 {

104 template <>

105 struct EnumMaxImpl<EngineCapability>

106 {

107

static constexpr int32 t kVALUE = 3;

108 };

109 } // namespace impl

10.14 NvInferRuntime.h

761

110

125

class Weights

126

{

127

public:

128

DataType type;

129

void const_* values;

130

int64 t count;

131

};

132

143

class IHostMemory : public INoCopy

144

{

145

public:

146

virtual ˜IHostMemory() noexcept = default;

147

149

void_* data() const noexcept

150

{

151

return mImpl->data();

152

}

153

155

std::size t size() const noexcept

156

{

157

return mImpl->size();

158

}

159

161

DataType type() const noexcept

162

{

163

return mImpl->type();

164

}

172

TRT DEPRECATED void destroy() noexcept

173

{

174

delete this;

175

}

176

177

protected:

178

apiv::VHostMemory* mImpl;

179

};

180

191

enum class DimensionOperation : int32 t

192

{

193

kSUM = 0,

194

kPROD = 1,

195

kMAX = 2,

196

kMIN = 3,

197

kSUB = 4,

198

kEQUAL = 5,

199

kLESS = 6,

200

kFLOOR DIV = 7,

201

kCEIL DIV = 8

202

};

203

205

template <>

206

constexpr inline int32 t EnumMax<DimensionOperation>() noexcept

207

{

208

return 9;

209

}

210

215

enum class TensorLocation : int32 t

216

{

217

kDEVICE = 0,

218

kHOST = 1,

219

};

220

221

namespace impl

222

{

224

template <>

225

struct EnumMaxImpl<TensorLocation>

226

{

227

static constexpr int32 t kVALUE = 2;

228

};

229

} // namespace impl

230

243

class IDimensionExpr : public INoCopy

244

{

245

public:

247

bool isConstant() const noexcept

248

{

249

return mImpl->isConstant();

250

}

251

254

int32 t getConstantValue() const noexcept

255

{

762

File Documentation

256

return mImpl->getConstantValue();

257

}

258

259

protected:

260

apiv::VDimensionExpr* mImpl;

261

virtual ˜IDimensionExpr() noexcept = default;

262

};

263

281

class IExprBuilder : public INoCopy

282

{

283

public:

285

IDimensionExpr const_* constant(int32 t value) noexcept

286

{

287

return mImpl->constant(value);

288

}

289

292

IDimensionExpr const_* operation(

293

DimensionOperation op, IDimensionExpr const& first, IDimensionExpr const& second) noexcept

294

{

295

return mImpl->operation(op, first, second);

296

}

297

298

protected:

299

apiv::VExprBuilder* mImpl;

300

virtual ˜IExprBuilder() noexcept = default;

301

};

302

308

class DimsExprs

309

{

310

public:

311

int32 t nbDims;

312

IDimensionExpr const* d[Dims::MAX DIMS];

313

};

314

320

struct DynamicPluginTensorDesc

321

{

323

PluginTensorDesc desc;

324

326

Dims min;

327

329

Dims max;

330

};

331

350

class IPluginV2DynamicExt : public nvinfer1::IPluginV2Ext

351

{

352

public:

353

IPluginV2DynamicExt_* clone() const noexcept override = 0;

354

379

virtual DimsExprs getOutputDimensions(

380

int32 t outputIndex, DimsExprs const* inputs, int32 t nbInputs, IExprBuilder& exprBuilder) noexcept =

381

385

static constexpr int32 t kFORMAT COMBINATION LIMIT = 100;

386

419

virtual bool supportsFormatCombination(

420

int32 t pos, PluginTensorDesc const* inOut, int32 t nbInputs, int32 t nbOutputs) noexcept = 0;

421

460

virtual void configurePlugin(DynamicPluginTensorDesc const* in, int32 t nbInputs,

461

DynamicPluginTensorDesc const* out, int32 t nbOutputs) noexcept = 0;

462

472

virtual size t getWorkspaceSize(PluginTensorDesc const* inputs, int32 t nbInputs, PluginTensorDesc const*

outputs,

473

int32 t nbOutputs) const noexcept = 0;

474

487

virtual int32 t enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc,

488

void const* const* inputs, void* const* outputs, void* workspace, cudaStream t stream) noexcept = 0;

489

490

protected:

498

int32 t getTensorRTVersion() const noexcept override

499

{

500

return (static cast<int32 t>(PluginVersion::kV2 DYNAMICEXT) << 24 | (NV TENSORRT VERSION & 0xFFFFFF));

501

}

502

503

virtual ˜IPluginV2DynamicExt() noexcept {}

504

505

private:

506

// Following are obsolete base class methods, and must not be implemented or used.

507

508

void configurePlugin(Dims const*, int32 t, Dims const*, int32 t, DataType const*, DataType const*, bool

const*,

509

bool const*, PluginFormat, int32 t) noexcept override final

10.14 NvInferRuntime.h

763

510

{

511

}

512

513

bool supportsFormat(DataType, PluginFormat) const noexcept override final

514

{

515

return false;

516

}

517

518

Dims getOutputDimensions(int32 t, Dims const*, int32 t) noexcept override final

519

{

520

return Dims{-1, {}};

521

}

522

523

bool isOutputBroadcastAcrossBatch(int32 t, bool const*, int32 t) const noexcept override final

524

{

525

return false;

526

}

527

528

bool canBroadcastInputAcrossBatch(int32 t) const noexcept override final

529

{

530

return true;

531

}

532

533

size t getWorkspaceSize(int32 t) const noexcept override final

534

{

535

return 0;

536

}

537

538

int32 t enqueue(int32 t, void const_* const*, void_* const*, void*, cudaStream t) noexcept override final

539

{

540

return 1;

541

}

542

};

543

555

class IProfiler

556

{

557

public:

565

virtual void reportLayerTime(char const* layerName, float ms) noexcept = 0;

566

567

virtual ˜IProfiler() noexcept {}

568

};

569

576

enum class WeightsRole : int32 t

577

{

578

kKERNEL = 0,

579

kBIAS = 1,

580

kSHIFT = 2,

581

kSCALE = 3,

582

kCONSTANT = 4,

583

kANY = 5,

584

};

585

587

template <>

588

constexpr inline int32 t EnumMax<WeightsRole>() noexcept

589

{

590

return 6;

591

}

592

598

enum class DeviceType : int32 t

599

{

600

kGPU,

601

kDLA,

602

};

603

605

template <>

606

constexpr inline int32 t EnumMax<DeviceType>() noexcept

607

{

608

return 2;

609

}

610

621

enum class TempfileControlFlag : int32 t

622

{

624

kALLOW IN MEMORY FILES = 0,

625

629

kALLOW TEMPORARY FILES = 1,

630

};

631

633

template <>

634

constexpr inline int32 t EnumMax<TempfileControlFlag>() noexcept

635

{

636

return 2;

764

File Documentation

637

}

638

644

using TempfileControlFlags = uint32 t;

645

653

class IRuntime : public INoCopy

654

{

655

public:

656

virtual ˜IRuntime() noexcept = default;

657

673

TRT DEPRECATED nvinfer1::ICudaEngine* deserializeCudaEngine(

674

void const* blob, std::size t size, IPluginFactory* pluginFactory) noexcept

675

{

676

return mImpl->deserializeCudaEngine(blob, size, nullptr);

677

}

678

689

void setDLACore(int32 t dlaCore) noexcept

690

{

691

mImpl->setDLACore(dlaCore);

692

}

693

698

int32 t getDLACore() const noexcept

699

{

700

return mImpl->getDLACore();

701

}

702

706

int32 t getNbDLACores() const noexcept

707

{

708

return mImpl->getNbDLACores();

709

}

710

718

TRT DEPRECATED void destroy() noexcept

719

{

720

delete this;

721

}

722

732

void setGpuAllocator(IGpuAllocator* allocator) noexcept

733

{

734

mImpl->setGpuAllocator(allocator);

735

}

736

748

751

void setErrorRecorder(IErrorRecorder* recorder) noexcept

752

{

753

mImpl->setErrorRecorder(recorder);

754

}

755

766

IErrorRecorder_* getErrorRecorder() const noexcept

767

{

768

return mImpl->getErrorRecorder();

769

}

770

781

ICudaEngine_* deserializeCudaEngine(void const* blob, std::size t size) noexcept

782

{

783

return mImpl->deserializeCudaEngine(blob, size, nullptr);

784

}

785

791

ILogger_* getLogger() const noexcept

792

{

793

return mImpl->getLogger();

794

}

795

805

bool setMaxThreads(int32 t maxThreads) noexcept

806

{

807

return mImpl->setMaxThreads(maxThreads);

808

}

809

819

int32 t getMaxThreads() const noexcept

820

{

821

return mImpl->getMaxThreads();

822

}

823

854

void setTemporaryDirectory(char const* path) noexcept

855

{

856

return mImpl->setTemporaryDirectory(path);

857

}

858

865

char const_* getTemporaryDirectory() const noexcept

866

{

867

return mImpl->getTemporaryDirectory();

868

}

869

10.14 NvInferRuntime.h

765

881

void setTempfileControlFlags(TempfileControlFlags flags) noexcept

882

{

883

return mImpl->setTempfileControlFlags(flags);

884

}

885

893

TempfileControlFlags getTempfileControlFlags() const noexcept

894

{

895

return mImpl->getTempfileControlFlags();

896

}

897

903

IPluginRegistry& getPluginRegistry() noexcept

904

{

905

return mImpl->getPluginRegistry();

906

}

907

921

IRuntime_* loadRuntime(char const* path) noexcept

922

{

923

return mImpl->loadRuntime(path);

924

}

925

933

void setEngineHostCodeAllowed(bool allowed) noexcept

934

{

935

return mImpl->setEngineHostCodeAllowed(allowed);

936

}

937

943

bool getEngineHostCodeAllowed() const noexcept

944

{

945

return mImpl->getEngineHostCodeAllowed();

946

}

947

948 protected:

949

apiv::VRuntime* mImpl;

950 };

951

959 class IRefitter : public INoCopy

960 {

961 public:

962

virtual ˜IRefitter() noexcept = default;

963

977

bool setWeights(char const* layerName, WeightsRole role, Weights weights) noexcept

978

{

979

return mImpl->setWeights(layerName, role, weights);

980

}

981

992

bool refitCudaEngine() noexcept

993

{

994

return mImpl->refitCudaEngine();

995

}

996

1013

int32 t getMissing(int32 t size, char const** layerNames, WeightsRole* roles) noexcept

1014

{

1015

return mImpl->getMissing(size, layerNames, roles);

1016

}

1017

1030

int32 t getAll(int32 t size, char const** layerNames, WeightsRole* roles) noexcept

1031

{

1032

return mImpl->getAll(size, layerNames, roles);

1033

}

1034

1040

TRT DEPRECATED void destroy() noexcept

1041

{

1042

delete this;

1043

}

1044

1060

bool setDynamicRange(char const* tensorName, float min, float max) noexcept

1061

{

1062

return mImpl->setDynamicRange(tensorName, min, max);

1063

}

1064

1074

float getDynamicRangeMin(char const* tensorName) const noexcept

1075

{

1076

return mImpl->getDynamicRangeMin(tensorName);

1077

}

1078

1088

float getDynamicRangeMax(char const* tensorName) const noexcept

1089

{

1090

return mImpl->getDynamicRangeMax(tensorName);

1091

}

1092

1104

int32 t getTensorsWithDynamicRange(int32 t size, char const** tensorNames) const noexcept

1105

{

766

File Documentation

1106

return mImpl->getTensorsWithDynamicRange(size, tensorNames);

1107

}

1108

1120

1123

void setErrorRecorder(IErrorRecorder* recorder) noexcept

1124

{

1125

mImpl->setErrorRecorder(recorder);

1126

}

1127

1138

IErrorRecorder_* getErrorRecorder() const noexcept

1139

{

1140

return mImpl->getErrorRecorder();

1141

}

1142

1159

bool setNamedWeights(char const* name, Weights weights) noexcept

1160

{

1161

return mImpl->setNamedWeights(name, weights);

1162

}

1163

1179

int32 t getMissingWeights(int32 t size, char const** weightsNames) noexcept

1180

{

1181

return mImpl->getMissingWeights(size, weightsNames);

1182

}

1183

1195

int32 t getAllWeights(int32 t size, char const** weightsNames) noexcept

1196

{

1197

return mImpl->getAllWeights(size, weightsNames);

1198

}

1199

1205

ILogger_* getLogger() const noexcept

1206

{

1207

return mImpl->getLogger();

1208

}

1209

1219

bool setMaxThreads(int32 t maxThreads) noexcept

1220

{

1221

return mImpl->setMaxThreads(maxThreads);

1222

}

1223

1233

int32 t getMaxThreads() const noexcept

1234

{

1235

return mImpl->getMaxThreads();

1236

}

1237

1238

protected:

1239

apiv::VRefitter* mImpl;

1240

};

1241

1252

enum class OptProfileSelector : int32 t

1253

{

1254

kMIN = 0,

1255

kOPT = 1,

1256

kMAX = 2

1257

};

1258

1264

template <>

1265

constexpr inline int32 t EnumMax<OptProfileSelector>() noexcept

1266

{

1267

return 3;

1268

}

1269

1292

class IOptimizationProfile : public INoCopy

1293

{

1294

public:

1322

bool setDimensions(char const* inputName, OptProfileSelector select, Dims dims) noexcept

1323

{

1324

return mImpl->setDimensions(inputName, select, dims);

1325

}

1326

1334

Dims getDimensions(char const* inputName, OptProfileSelector select) const noexcept

1335

{

1336

return mImpl->getDimensions(inputName, select);

1337

}

1338

1380

bool setShapeValues(

1381

char const* inputName, OptProfileSelector select, int32 t const* values, int32 t nbValues) noexcept

1382

{

1383

return mImpl->setShapeValues(inputName, select, values, nbValues);

1384

}

1385

1394

int32 t getNbShapeValues(char const* inputName) const noexcept

10.14 NvInferRuntime.h

767

1395

{

1396

return mImpl->getNbShapeValues(inputName);

1397

}

1398

1406

int32 t const_* getShapeValues(char const* inputName, OptProfileSelector select) const noexcept

1407

{

1408

return mImpl->getShapeValues(inputName, select);

1409

}

1410

1424

bool setExtraMemoryTarget(float target) noexcept

1425

{

1426

return mImpl->setExtraMemoryTarget(target);

1427

}

1428

1436

float getExtraMemoryTarget() const noexcept

1437

{

1438

return mImpl->getExtraMemoryTarget();

1439

}

1440

1453

bool isValid() const noexcept

1454

{

1455

return mImpl->isValid();

1456

}

1457

1458

protected:

1459

apiv::VOptimizationProfile* mImpl;

1460

virtual ˜IOptimizationProfile() noexcept = default;

1461

};

1462

1471

enum class TacticSource : int32 t

1472

{

1475

kCUBLAS = 0,

1478

kCUBLAS LT = 1,

1481

kCUDNN = 2,

1482

1486

kEDGE MASK CONVOLUTIONS = 3,

1487

1490

kJIT CONVOLUTIONS = 4,

1491

};

1492

1493

template <>

1494

constexpr inline int32 t EnumMax<TacticSource>() noexcept

1495

{

1496

return 5;

1497

}

1498

1505

using TacticSources = uint32 t;

1506

1516

enum class ProfilingVerbosity : int32 t

1517

{

1518

kLAYER NAMES ONLY = 0,

1519

kNONE = 1,

1520

kDETAILED = 2,

1521

1523

kDEFAULT TRT DEPRECATED ENUM = kLAYER NAMES ONLY,

1525

kVERBOSE TRT DEPRECATED ENUM = kDETAILED

1526

};

1527

1529

template <>

1530

constexpr inline int32 t EnumMax<ProfilingVerbosity>() noexcept

1531

{

1532

return 3;

1533

}

1534

1542

class ICudaEngine : public INoCopy

1543

{

1544

public:

1545

virtual ˜ICudaEngine() noexcept = default;

1546

1559

TRT DEPRECATED int32 t getNbBindings() const noexcept

1560

{

1561

return mImpl->getNbBindings();

1562

}

1563

1587

TRT DEPRECATED int32 t getBindingIndex(char const* name) const noexcept

1588

{

1589

return mImpl->getBindingIndex(name);

1590

}

1591

1610

TRT DEPRECATED char const_* getBindingName(int32 t bindingIndex) const noexcept

1611

{

768

File Documentation

1612

return mImpl->getBindingName(bindingIndex);

1613

}

1614

1625

TRT DEPRECATED bool bindingIsInput(int32 t bindingIndex) const noexcept

1626

{

1627

return mImpl->bindingIsInput(bindingIndex);

1628

}

1629

1652

TRT DEPRECATED Dims getBindingDimensions(int32 t bindingIndex) const noexcept

1653

{

1654

return mImpl->getBindingDimensions(bindingIndex);

1655

}

1656

1667

Dims getTensorShape(char const* tensorName) const noexcept

1668

{

1669

return mImpl->getTensorShape(tensorName);

1670

}

1671

1682

TRT DEPRECATED DataType getBindingDataType(int32 t bindingIndex) const noexcept

1683

{

1684

return mImpl->getBindingDataType(bindingIndex);

1685

}

1686

1697

DataType getTensorDataType(char const* tensorName) const noexcept

1698

{

1699

return mImpl->getTensorDataType(tensorName);

1700

}

1701

1713

TRT DEPRECATED int32 t getMaxBatchSize() const noexcept

1714

{

1715

return mImpl->getMaxBatchSize();

1716

}

1717

1727

int32 t getNbLayers() const noexcept

1728

{

1729

return mImpl->getNbLayers();

1730

}

1731

1741

IHostMemory_* serialize() const noexcept

1742

{

1743

return mImpl->serialize();

1744

}

1745

1758

IExecutionContext_* createExecutionContext() noexcept

1759

{

1760

return mImpl->createExecutionContext();

1761

}

1762

1770

TRT DEPRECATED void destroy() noexcept

1771

{

1772

delete this;

1773

}

1774

1788

TRT DEPRECATED TensorLocation getLocation(int32 t bindingIndex) const noexcept

1789

{

1790

return mImpl->getLocation(bindingIndex);

1791

}

1792

1805

TensorLocation getTensorLocation(char const* tensorName) const noexcept

1806

{

1807

return mImpl->getTensorLocation(tensorName);

1808

}

1809

1825

bool isShapeInferenceIO(char const* tensorName) const noexcept

1826

{

1827

return mImpl->isShapeInferenceIO(tensorName);

1828

}

1829

1839

TensorIOMode getTensorIOMode(char const* tensorName) const noexcept

1840

{

1841

return mImpl->getTensorIOMode(tensorName);

1842

}

1843

1848

IExecutionContext_* createExecutionContextWithoutDeviceMemory() noexcept

1849

{

1850

return mImpl->createExecutionContextWithoutDeviceMemory();

1851

}

1852

1858

size t getDeviceMemorySize() const noexcept

1859

{

1860

return mImpl->getDeviceMemorySize();

10.14 NvInferRuntime.h

769

1861

}

1862

1868

bool isRefittable() const noexcept

1869

{

1870

return mImpl->isRefittable();

1871

}

1872

1885

TRT DEPRECATED int32 t getBindingBytesPerComponent(int32 t bindingIndex) const noexcept

1886

{

1887

return mImpl->getBindingBytesPerComponent(bindingIndex);

1888

}

1889

1906

int32 t getTensorBytesPerComponent(char const* tensorName) const noexcept

1907

{

1908

return mImpl->getTensorBytesPerComponent(tensorName);

1909

}

1910

1924

int32 t getTensorBytesPerComponent(char const* tensorName, int32 t profileIndex) const noexcept

1925

{

1926

return mImpl->getTensorBytesPerComponentV2(tensorName, profileIndex);

1927

}

1928

1940

TRT DEPRECATED int32 t getBindingComponentsPerElement(int32 t bindingIndex) const noexcept

1941

{

1942

return mImpl->getBindingComponentsPerElement(bindingIndex);

1943

}

1944

1961

int32 t getTensorComponentsPerElement(char const* tensorName) const noexcept

1962

{

1963

return mImpl->getTensorComponentsPerElement(tensorName);

1964

}

1965

1979

int32 t getTensorComponentsPerElement(char const* tensorName, int32 t profileIndex) const noexcept

1980

{

1981

return mImpl->getTensorComponentsPerElementV2(tensorName, profileIndex);

1982

}

1983

1993

TRT DEPRECATED TensorFormat getBindingFormat(int32 t bindingIndex) const noexcept

1994

{

1995

return mImpl->getBindingFormat(bindingIndex);

1996

}

1997

2008

TensorFormat getTensorFormat(char const* tensorName) const noexcept

2009

{

2010

return mImpl->getTensorFormat(tensorName);

2011

}

2012

2022

TensorFormat getTensorFormat(char const* tensorName, int32 t profileIndex) const noexcept

2023

{

2024

return mImpl->getTensorFormatV2(tensorName, profileIndex);

2025

}

2026

2046

TRT DEPRECATED char const_* getBindingFormatDesc(int32 t bindingIndex) const noexcept

2047

{

2048

return mImpl->getBindingFormatDesc(bindingIndex);

2049

}

2050

2070

char const_* getTensorFormatDesc(char const* tensorName) const noexcept

2071

{

2072

return mImpl->getTensorFormatDesc(tensorName);

2073

}

2074

2093

char const_* getTensorFormatDesc(char const* tensorName, int32 t profileIndex) const noexcept

2094

{

2095

return mImpl->getTensorFormatDescV2(tensorName, profileIndex);

2096

}

2097

2109

TRT DEPRECATED int32 t getBindingVectorizedDim(int32 t bindingIndex) const noexcept

2110

{

2111

return mImpl->getBindingVectorizedDim(bindingIndex);

2112

}

2113

2126

int32 t getTensorVectorizedDim(char const* tensorName) const noexcept

2127

{

2128

return mImpl->getTensorVectorizedDim(tensorName);

2129

}

2130

2142

int32 t getTensorVectorizedDim(char const* tensorName, int32 t profileIndex) const noexcept

2143

{

2144

return mImpl->getTensorVectorizedDimV2(tensorName, profileIndex);

2145

}

770

File Documentation

2146

2157

char const_* getName() const noexcept

2158

{

2159

return mImpl->getName();

2160

}

2161

2168

int32 t getNbOptimizationProfiles() const noexcept

2169

{

2170

return mImpl->getNbOptimizationProfiles();

2171

}

2172

2200

TRT DEPRECATED Dims getProfileDimensions(

2201

int32 t bindingIndex, int32 t profileIndex, OptProfileSelector select) const noexcept

2202

{

2203

return mImpl->getProfileDimensions(bindingIndex, profileIndex, select);

2204

}

2205

2221

Dims getProfileShape(char const* tensorName, int32 t profileIndex, OptProfileSelector select) const

noexcept

2222

{

2223

return mImpl->getProfileShape(tensorName, profileIndex, select);

2224

}

2225

2250

TRT DEPRECATED int32 t const_* getProfileShapeValues(

2251

int32 t profileIndex, int32 t inputIndex, OptProfileSelector select) const noexcept

2252

{

2253

return mImpl->getProfileShapeValues(profileIndex, inputIndex, select);

2254

}

2255

2289

TRT DEPRECATED bool isShapeBinding(int32 t bindingIndex) const noexcept

2290

{

2291

return mImpl->isShapeBinding(bindingIndex);

2292

}

2293

2306

TRT DEPRECATED bool isExecutionBinding(int32 t bindingIndex) const noexcept

2307

{

2308

return mImpl->isExecutionBinding(bindingIndex);

2309

}

2310

2321

EngineCapability getEngineCapability() const noexcept

2322

{

2323

return mImpl->getEngineCapability();

2324

}

2325

2336

2339

void setErrorRecorder(IErrorRecorder* recorder) noexcept

2340

{

2341

return mImpl->setErrorRecorder(recorder);

2342

}

2343

2354

IErrorRecorder_* getErrorRecorder() const noexcept

2355

{

2356

return mImpl->getErrorRecorder();

2357

}

2358

2373

bool hasImplicitBatchDimension() const noexcept

2374

{

2375

return mImpl->hasImplicitBatchDimension();

2376

}

2377

2388

TacticSources getTacticSources() const noexcept

2389

{

2390

return mImpl->getTacticSources();

2391

}

2392

2399

ProfilingVerbosity getProfilingVerbosity() const noexcept

2400

{

2401

return mImpl->getProfilingVerbosity();

2402

}

2403

2409

IEngineInspector_* createEngineInspector() const noexcept

2410

{

2411

return mImpl->createEngineInspector();

2412

}

2413

2422

int32 t getNbIOTensors() const noexcept

2423

{

2424

return mImpl->getNbIOTensors();

2425

}

2426

2434

char const_* getIOTensorName(int32 t index) const noexcept

10.14 NvInferRuntime.h

771

2435

{

2436

return mImpl->getIOTensorName(index);

2437

}

2438

2446

HardwareCompatibilityLevel getHardwareCompatibilityLevel() const noexcept

2447

{

2448

return mImpl->getHardwareCompatibilityLevel();

2449

}

2450

2461

int32 t getNbAuxStreams() const noexcept

2462

{

2463

return mImpl->getNbAuxStreams();

2464

}

2465

2466

protected:

2467

apiv::VCudaEngine* mImpl;

2468

};

2469

2479

class IOutputAllocator

2480

{

2481

public:

2489

virtual int32 t getInterfaceVersion() const noexcept

2490

{

2491

return 1;

2492

}

2493

2510

virtual void_* reallocateOutput(char const* tensorName, void* currentMemory, uint64 t size, uint64 t

alignment) noexcept = 0;

2511

2520

virtual void notifyShape(char const* tensorName, Dims const& dims) noexcept = 0;

2521

2522

virtual ˜IOutputAllocator() = default;

2523

};

2524

2535

class IExecutionContext : public INoCopy

2536

{

2537

public:

2538

virtual ˜IExecutionContext() noexcept = default;

2539

2562

TRT DEPRECATED bool execute(int32 t batchSize, void* const* bindings) noexcept

2563

{

2564

return mImpl->execute(batchSize, bindings);

2565

}

2566

2596

TRT DEPRECATED bool enqueue(

2597

int32 t batchSize, void_* const* bindings, cudaStream t stream, cudaEvent t* inputConsumed) noexcept

2598

{

2599

return mImpl->enqueue(batchSize, bindings, stream, inputConsumed);

2600

}

2601

2610

void setDebugSync(bool sync) noexcept

2611

{

2612

mImpl->setDebugSync(sync);

2613

}

2614

2620

bool getDebugSync() const noexcept

2621

{

2622

return mImpl->getDebugSync();

2623

}

2624

2630

void setProfiler(IProfiler* profiler) noexcept

2631

{

2632

mImpl->setProfiler(profiler);

2633

}

2634

2640

IProfiler_* getProfiler() const noexcept

2641

{

2642

return mImpl->getProfiler();

2643

}

2644

2650

ICudaEngine const& getEngine() const noexcept

2651

{

2652

return mImpl->getEngine();

2653

}

2654

2662

TRT DEPRECATED void destroy() noexcept

2663

{

2664

delete this;

2665

}

2666

2676

void setName(char const* name) noexcept

772

File Documentation

2677

{

2678

mImpl->setName(name);

2679

}

2680

2686

char const_* getName() const noexcept

2687

{

2688

return mImpl->getName();

2689

}

2690

2703

void setDeviceMemory(void* memory) noexcept

2704

{

2705

mImpl->setDeviceMemory(memory);

2706

}

2707

2726

TRT DEPRECATED Dims getStrides(int32 t bindingIndex) const noexcept

2727

{

2728

return mImpl->getStrides(bindingIndex);

2729

}

2730

2747

Dims getTensorStrides(char const* tensorName) const noexcept

2748

{

2749

return mImpl->getTensorStrides(tensorName);

2750

}

2751

2752

public:

2784

TRT DEPRECATED

2785

bool setOptimizationProfile(int32 t profileIndex) noexcept

2786

{

2787

return mImpl->setOptimizationProfile(profileIndex);

2788

}

2789

2799

int32 t getOptimizationProfile() const noexcept

2800

{

2801

return mImpl->getOptimizationProfile();

2802

}

2803

2838

TRT DEPRECATED bool setBindingDimensions(int32 t bindingIndex, Dims dimensions) noexcept

2839

{

2840

return mImpl->setBindingDimensions(bindingIndex, dimensions);

2841

}

2842

2856

bool setInputShape(char const* tensorName, Dims const& dims) noexcept

2857

{

2858

return mImpl->setInputShape(tensorName, dims);

2859

}

2860

2889

TRT DEPRECATED Dims getBindingDimensions(int32 t bindingIndex) const noexcept

2890

{

2891

return mImpl->getBindingDimensions(bindingIndex);

2892

}

2893

2926

Dims getTensorShape(char const* tensorName) const noexcept

2927

{

2928

return mImpl->getTensorShape(tensorName);

2929

}

2930

2961

TRT DEPRECATED bool setInputShapeBinding(int32 t bindingIndex, int32 t const* data) noexcept

2962

{

2963

return mImpl->setInputShapeBinding(bindingIndex, data);

2964

}

2965

2985

TRT DEPRECATED bool getShapeBinding(int32 t bindingIndex, int32 t* data) const noexcept

2986

{

2987

return mImpl->getShapeBinding(bindingIndex, data);

2988

}

2989

3003

bool allInputDimensionsSpecified() const noexcept

3004

{

3005

return mImpl->allInputDimensionsSpecified();

3006

}

3007

3020

bool allInputShapesSpecified() const noexcept

3021

{

3022

return mImpl->allInputShapesSpecified();

3023

}

3024

3036

3039

void setErrorRecorder(IErrorRecorder* recorder) noexcept

3040

{

3041

mImpl->setErrorRecorder(recorder);

3042

}

10.14 NvInferRuntime.h

773

3043

3054

IErrorRecorder_* getErrorRecorder() const noexcept

3055

{

3056

return mImpl->getErrorRecorder();

3057

}

3058

3071

bool executeV2(void_* const* bindings) noexcept

3072

{

3073

return mImpl->executeV2(bindings);

3074

}

3075

3101

TRT DEPRECATED bool enqueueV2(void_* const* bindings, cudaStream t stream, cudaEvent t* inputConsumed)

noexcept

3102

{

3103

return mImpl->enqueueV2(bindings, stream, inputConsumed);

3104

}

3105

3148

bool setOptimizationProfileAsync(int32 t profileIndex, cudaStream t stream) noexcept

3149

{

3150

return mImpl->setOptimizationProfileAsync(profileIndex, stream);

3151

}

3152

3163

void setEnqueueEmitsProfile(bool enqueueEmitsProfile) noexcept

3164

{

3165

mImpl->setEnqueueEmitsProfile(enqueueEmitsProfile);

3166

}

3167

3174

bool getEnqueueEmitsProfile() const noexcept

3175

{

3176

return mImpl->getEnqueueEmitsProfile();

3177

}

3178

3203

bool reportToProfiler() const noexcept

3204

{

3205

return mImpl->reportToProfiler();

3206

}

3207

3245

bool setTensorAddress(char const* tensorName, void* data) noexcept

3246

{

3247

return mImpl->setTensorAddress(tensorName, data);

3248

}

3249

3262

void const_* getTensorAddress(char const* tensorName) const noexcept

3263

{

3264

return mImpl->getTensorAddress(tensorName);

3265

}

3266

3284

bool setInputTensorAddress(char const* tensorName, void const* data) noexcept

3285

{

3286

return mImpl->setInputTensorAddress(tensorName, data);

3287

}

3288

3303

void_* getOutputTensorAddress(char const* tensorName) const noexcept

3304

{

3305

return mImpl->getOutputTensorAddress(tensorName);

3306

}

3307

3336

int32 t inferShapes(int32 t nbMaxNames, char const** tensorNames) noexcept

3337

{

3338

return mImpl->inferShapes(nbMaxNames, tensorNames);

3339

}

3340

3352

bool setInputConsumedEvent(cudaEvent t event) noexcept

3353

{

3354

return mImpl->setInputConsumedEvent(event);

3355

}

3356

3362

cudaEvent t getInputConsumedEvent() const noexcept

3363

{

3364

return mImpl->getInputConsumedEvent();

3365

}

3366

3381

bool setOutputAllocator(char const* tensorName, IOutputAllocator* outputAllocator) noexcept

3382

{

3383

return mImpl->setOutputAllocator(tensorName, outputAllocator);

3384

}

3385

3394

IOutputAllocator_* getOutputAllocator(char const* tensorName) const noexcept

3395

{

3396

return mImpl->getOutputAllocator(tensorName);

3397

}

774

File Documentation

3398

3412

int64 t getMaxOutputSize(char const* tensorName) const noexcept

3413

{

3414

return mImpl->getMaxOutputSize(tensorName);

3415

}

3416

3432

bool setTemporaryStorageAllocator(IGpuAllocator* allocator) noexcept

3433

{

3434

return mImpl->setTemporaryStorageAllocator(allocator);

3435

}

3436

3442

IGpuAllocator_* getTemporaryStorageAllocator() const noexcept

3443

{

3444

return mImpl->getTemporaryStorageAllocator();

3445

}

3446

3460

bool enqueueV3(cudaStream t stream) noexcept

3461

{

3462

return mImpl->enqueueV3(stream);

3463

}

3464

3475

void setPersistentCacheLimit(size t size) noexcept

3476

{

3477

mImpl->setPersistentCacheLimit(size);

3478

}

3479

3486

size t getPersistentCacheLimit() const noexcept

3487

{

3488

return mImpl->getPersistentCacheLimit();

3489

}

3490

3510

bool setNvtxVerbosity(ProfilingVerbosity verbosity) noexcept

3511

{

3512

return mImpl->setNvtxVerbosity(verbosity);

3513

}

3514

3522

ProfilingVerbosity getNvtxVerbosity() const noexcept

3523

{

3524

return mImpl->getNvtxVerbosity();

3525

}

3526

3553

void setAuxStreams(cudaStream t* auxStreams, int32 t nbStreams) noexcept

3554

{

3555

mImpl->setAuxStreams(auxStreams, nbStreams);

3556

}

3557

3558

protected:

3559

apiv::VExecutionContext* mImpl;

3560

}; // class IExecutionContext

3561

3569

enum class LayerInformationFormat : int32 t

3570

{

3571

kONELINE = 0,

3572

kJSON = 1,

3573

};

3574

3577

template <>

3578

constexpr inline int32 t EnumMax<LayerInformationFormat>() noexcept

3579

{

3580

return 2;

3581

}

3582

3598

class IEngineInspector : public INoCopy

3599

{

3600

public:

3601

virtual ˜IEngineInspector() noexcept = default;

3602

3615

bool setExecutionContext(IExecutionContext const* context) noexcept

3616

{

3617

return mImpl->setExecutionContext(context);

3618

}

3619

3627

IExecutionContext const_* getExecutionContext() const noexcept

3628

{

3629

return mImpl->getExecutionContext();

3630

}

3631

3652

char const_* getLayerInformation(int32 t layerIndex, LayerInformationFormat format) const noexcept

3653

{

3654

return mImpl->getLayerInformation(layerIndex, format);

3655

}

10.14 NvInferRuntime.h

775

3656

3675

char const_* getEngineInformation(LayerInformationFormat format) const noexcept

3676

{

3677

return mImpl->getEngineInformation(format);

3678

}

3679

3691

3694

void setErrorRecorder(IErrorRecorder* recorder) noexcept

3695

{

3696

mImpl->setErrorRecorder(recorder);

3697

}

3698

3709

IErrorRecorder_* getErrorRecorder() const noexcept

3710

{

3711

return mImpl->getErrorRecorder();

3712

}

3713

3714

protected:

3715

apiv::VEngineInspector* mImpl;

3716

}; // class IEngineInspector

3717

3718

} // namespace nvinfer1

3719

3724

extern "C" TENSORRTAPI void* createInferRuntime INTERNAL(void* logger, int32 t version) noexcept;

3725

3730

extern "C" TENSORRTAPI void* createInferRefitter INTERNAL(void* engine, void* logger, int32 t version)

noexcept;

3731

3735

extern "C" TENSORRTAPI nvinfer1::IPluginRegistry_* getPluginRegistry() noexcept;

3736

3742

extern "C" TENSORRTAPI nvinfer1::ILogger* getLogger() noexcept;

3743

3744

namespace nvinfer1

3745 {

3746

namespace // unnamed namespace avoids linkage surprises when linking objects built with different versions

of this

3747

// header.

3748 {

3754

inline IRuntime* createInferRuntime(ILogger& logger) noexcept

3755 {

3756

return static cast<IRuntime*>(createInferRuntime INTERNAL(&logger, NV TENSORRT VERSION));

3757 }

3758

3764

inline IRefitter* createInferRefitter(ICudaEngine& engine, ILogger& logger) noexcept

3765 {

3766

return static cast<IRefitter*>(createInferRefitter INTERNAL(&engine, &logger, NV TENSORRT VERSION));

3767 }

3768

3769

} // namespace

3770

3782

template <typename T>

3783

class PluginRegistrar

3784 {

3785

public:

3786

PluginRegistrar()

3787

{

3788

getPluginRegistry()->registerCreator(instance, "");

3789

}

3790

3791

private:

3793

T instance{};

3794 };

3795

3796

} // namespace nvinfer1

3797

3798

#define REGISTER TENSORRT PLUGIN(name)

3799

static nvinfer1::PluginRegistrar<name> pluginRegistrar##name {}

3800

3801

namespace nvinfer1

3802

{

3811

class ILoggerFinder

3812

{

3813

public:

3821

virtual ILogger_* findLogger() = 0;

3822

3823

protected:

3824

virtual ˜ILoggerFinder() = default;

3825

};

3826

3827

} // namespace nvinfer1

776

File Documentation

3828

3829 #endif // NV INFER RUNTIME H

10.15

NvInferRuntimeBase.h File Reference

#include "NvInferVersion.h"

#include <cstddef>

#include <cstdint>

#include <cuda runtime api.h>

Classes

• struct nvinfer1::impl::EnumMaxImpl< DataType >

Maximum number of elements in DataType enum.

• class nvinfer1::Dims32

• struct nvinfer1::impl::EnumMaxImpl< TensorFormat >

Maximum number of elements in TensorFormat enum.

• struct nvinfer1::impl::EnumMaxImpl< AllocatorFlag >

Maximum number of elements in AllocatorFlag enum.

• class nvinfer1::IGpuAllocator

Application-implemented class for controlling allocation on the GPU.

• class nvinfer1::ILogger

Application-implemented logging interface for the builder, refitter and runtime.

• struct nvinfer1::impl::EnumMaxImpl< ILogger::Severity >

Maximum number of elements in ILogger::Severity enum.

• struct nvinfer1::impl::EnumMaxImpl< ErrorCode >

Maximum number of elements in ErrorCode enum.

• class nvinfer1::IErrorRecorder

Reference counted application-implemented error reporting interface for TensorRT objects.

• struct nvinfer1::impl::EnumMaxImpl< TensorIOMode >

Maximum number of elements in TensorIOMode enum.

Namespaces

• namespace nvinfer1

The TensorRT API version 1 namespace.

• namespace nvinfer1::impl

Macros

• #define TRT DEPRECATED attribute

((deprecated))

• #define TRT DEPRECATED ENUM

• #define TRT DEPRECATED API attribute

((deprecated, visibility(”default”)))

• #define TENSORRTAPI

• #define TRTNOEXCEPT

• #define NV TENSORRT VERSION nvinfer1::kNV TENSORRT VERSION IMPL

10.15 NvInferRuntimeBase.h File Reference

777

Typedefs

• using nvinfer1::char t = char

char t is the type used by TensorRT to represent all valid characters.

• using nvinfer1::AsciiChar = char t

• using nvinfer1::Dims = Dims32

• using nvinfer1::AllocatorFlags = uint32 t

Enumerations

• enum class nvinfer1::DataType : int32 t {

nvinfer1::kFLOAT = 0 , nvinfer1::kHALF = 1 , nvinfer1::kINT8 = 2 , nvinfer1::kINT32 = 3 ,

nvinfer1::kBOOL = 4 , nvinfer1::kUINT8 = 5 , nvinfer1::kFP8 = 6 }

The type of weights and tensors.

• enum class nvinfer1::TensorFormat : int32 t {

nvinfer1::kLINEAR = 0 , nvinfer1::kCHW2 = 1 , nvinfer1::kHWC8 = 2 , nvinfer1::kCHW4 = 3 ,

nvinfer1::kCHW16 = 4 , nvinfer1::kCHW32 = 5 , nvinfer1::kDHWC8 = 6 , nvinfer1::kCDHW32 = 7 ,

nvinfer1::kHWC = 8 , nvinfer1::kDLA LINEAR = 9 , nvinfer1::kDLA HWC4 = 10 , nvinfer1::kHWC16 = 11 ,

nvinfer1::kDHWC = 12 }

Format of the input/output tensors.

• enum class nvinfer1::AllocatorFlag : int32 t { nvinfer1::kRESIZABLE = 0 }

• enum class nvinfer1::ErrorCode : int32 t {

nvinfer1::kSUCCESS = 0 , nvinfer1::kUNSPECIFIED ERROR = 1 , nvinfer1::kINTERNAL ERROR = 2 ,

nvinfer1::kINVALID ARGUMENT = 3 ,

nvinfer1::kINVALID CONFIG = 4 , nvinfer1::kFAILED ALLOCATION = 5 , nvinfer1::kFAILED INITIALIZATION

= 6 , nvinfer1::kFAILED EXECUTION = 7 ,

nvinfer1::kFAILED COMPUTATION = 8 , nvinfer1::kINVALID STATE = 9 , nvinfer1::kUNSUPPORTED STATE

= 10 }

Error codes that can be returned by TensorRT during execution.

• enum class nvinfer1::TensorIOMode

int32 t { nvinfer1::kNONE

, nvinfer1::kINPUT

nvinfer1::kOUTPUT = 2 }

Definition of tensor IO Mode.

Functions

• template<typename T >

constexpr int32 t nvinfer1::EnumMax () noexcept

Maximum number of elements in an enumeration type.

• int32 t getInferLibVersion () noexcept

Return the library version number.

10.15.1

Detailed Description

This file contains common definitions, data structures and interfaces shared between the standard and safe runtime.

10.16 NvInferRuntimeBase.h

779

10.15.3

Function Documentation

10.15.3.1 getInferLibVersion()

int32 t getInferLibVersion ( ) [noexcept]

Return the library version number.

The format is as for TENSORRT VERSION: (TENSORRT MAJOR ∗ 1000) + (TENSORRT MINOR ∗ 100) +

TENSOR PATCH.

10.16

NvInferRuntimeBase.h

Go to the documentation of this file.

1 /*

* SPDX-License-Identifier: LicenseRef-NvidiaProprietary

* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual

* property and proprietary rights in and to this material, related

* documentation and any modifications thereto. Any use, reproduction,

* disclosure or distribution of this material and related documentation

* without an express license agreement from NVIDIA CORPORATION or

* its affiliates is strictly prohibited.

#ifndef NV INFER RUNTIME BASE H

#define NV INFER RUNTIME BASE H

#include "NvInferVersion.h"

#include <cstddef>

#include <cstdint>

#include <cuda runtime api.h>

// Items that are marked as deprecated will be removed in a future release.

#if

cplusplus >= 201402L

#define TRT DEPRECATED [[deprecated]]

#if

GNUC

< 6

#define TRT DEPRECATED ENUM

#else

#define TRT DEPRECATED ENUM TRT DEPRECATED

#endif

#ifdef MSC VER

#define TRT DEPRECATED API

declspec(dllexport)

#else

#define TRT DEPRECATED API [[deprecated]]

attribute ((visibility("default")))

#endif

#else

#ifdef MSC VER

#define TRT DEPRECATED

#define TRT DEPRECATED ENUM

#define TRT DEPRECATED API

declspec(dllexport)

#else

#define TRT DEPRECATED

attribute ((deprecated))

#define TRT DEPRECATED ENUM

#define TRT DEPRECATED API

attribute ((deprecated, visibility("default")))

#endif

// Defines which symbols are exported

#ifdef TENSORRT BUILD LIB

#ifdef MSC VER

#define TENSORRTAPI

declspec(dllexport)

#else

780

File Documentation

51 #define TENSORRTAPI

attribute ((visibility("default")))

52 #endif

53 #else

54 #define TENSORRTAPI

55 #endif

56 #define TRTNOEXCEPT

66 // forward declare some CUDA types to avoid an include dependency

68 extern "C"

69 {

struct cublasContext;

struct cudnnContext;

74 }

76 #define NV TENSORRT VERSION nvinfer1::kNV TENSORRT VERSION IMPL

82 namespace nvinfer1

83 {

85 static constexpr int32 t kNV TENSORRT VERSION IMPL

= (NV TENSORRT MAJOR * 1000) + (NV TENSORRT MINOR * 100) + NV TENSORRT PATCH; // major, minor, patch

89 using char t = char;

94 using AsciiChar = char t;

97 class IErrorRecorder;

99 class IGpuAllocator;

100

101

namespace impl

102

{

104

template <typename T>

105

struct EnumMaxImpl;

106

} // namespace impl

107

109

template <typename T>

110

constexpr int32 t EnumMax() noexcept

111

{

112

return impl::EnumMaxImpl<T>::kVALUE;

113

}

114

119

enum class DataType : int32 t

120

{

122

kFLOAT = 0,

123

125

kHALF = 1,

126

128

kINT8 = 2,

129

131

kINT32 = 3,

132

134

kBOOL = 4,

135

148

kUINT8 = 5,

149

153

kFP8 = 6

154

155 };

156

157

namespace impl

158 {

160

template <>

161

struct EnumMaxImpl<DataType>

162 {

163

// Declaration of kVALUE that represents maximum number of elements in DataType enum

164

static constexpr int32 t kVALUE = 7;

165 };

166

} // namespace impl

167

178

class Dims32

179 {

180

public:

182

static constexpr int32 t MAX DIMS{8};

184

int32 t nbDims;

186

int32 t d[MAX DIMS];

187 };

188

194

using Dims = Dims32;

195

208

enum class TensorFormat : int32 t

209 {

10.16 NvInferRuntimeBase.h

781

217

kLINEAR = 0,

218

225

kCHW2 = 1,

226

233

kHWC8 = 2,

234

250

kCHW4 = 3,

251

262

kCHW16 = 4,

263

273

kCHW32 = 5,

274

281

kDHWC8 = 6,

282

289

kCDHW32 = 7,

290

293

kHWC = 8,

294

303

kDLA LINEAR = 9,

304

318

kDLA HWC4 = 10,

319

326

kHWC16 = 11,

327

330

kDHWC = 12

331

};

332

333

namespace impl

334

{

336

template <>

337

struct EnumMaxImpl<TensorFormat>

338

{

340

static constexpr int32 t kVALUE = 13;

341

};

342

} // namespace impl

343

344

enum class AllocatorFlag : int32 t

345

{

346

kRESIZABLE = 0,

347

};

348

349

namespace impl

350

{

352

template <>

353

struct EnumMaxImpl<AllocatorFlag>

354

{

355

static constexpr int32 t kVALUE = 1;

356

};

357

} // namespace impl

358

359

using AllocatorFlags = uint32 t;

360

366

class IGpuAllocator

367

{

368

public:

390

virtual void_* allocate(uint64 t const size, uint64 t const alignment, AllocatorFlags const flags)

noexcept = 0;

391

410

TRT DEPRECATED virtual void free(void_* const memory) noexcept = 0;

411

416

virtual ˜IGpuAllocator() = default;

417

IGpuAllocator() = default;

418

452

virtual void_* reallocate(void_* /*baseAddr*/, uint64 t /*alignment*/, uint64 t /*newSize*/) noexcept

453

{

454

return nullptr;

455

}

456

477

virtual bool deallocate(void_* const memory) noexcept

478

{

479

this->free(memory);

480

return true;

481

}

482

483

protected:

484

// @cond SuppressDoxyWarnings

485

IGpuAllocator(IGpuAllocator const&) = default;

486

IGpuAllocator(IGpuAllocator&&) = default;

487

IGpuAllocator& operator=(IGpuAllocator const&) & = default;

488

IGpuAllocator& operator=(IGpuAllocator&&) & = default;

489

// @endcond

NVIDIA TensorRT 8.6.11 API Reference for DRIVE OS (May 2023) - page 11